Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dslify quant matmul

eugene2e1014428f0f
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
index ad92a5179..62f28b658 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
@@ -5,11 +5,11 @@ use super::EmbeddingError;
use crate::{
DataType,
backends::{
- common::Context,
- metal::{
- Metal, MetalContext, MetalError,
- kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},
+ common::{
+ Context,
+ kernel::matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulType},
},
+ metal::{Metal, MetalContext, MetalError, kernel::quant_matmul::QuantizedMatmulKernel},
},
config::QuantizationMode,
encodable_block::{EncodableBlock, EncodingParameters},
@@ -161,13 +161,15 @@ impl QuantizedEmbeddingReadout {
let kernel = QuantizedMatmulKernel::new(
mtl_context,
- data_type,
- group_size,
- model_dim,
- vocab_size,
- mode,
- QuantizationType::Mlx,
- weights_transposed,
+ QuantizedMatmulConfiguration {
+ data_type,
+ group_size,
+ input_dim: model_dim,
+ output_dim: vocab_size,
+ mode,
+ quantization_type: QuantizedMatmulType::Mlx,
+ weights_transposed,
+ },
)
.map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to create kernel: {:?}", e))))?;
@@ -225,15 +227,15 @@ impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {
let args = QuantizedMatmulArguments {
a_buffer: input_buffer,
- a_offset,
+ a_offset: a_offset as usize,
b_buffer: &self.weights_buffer,
scales_buffer: &self.scales_buffer,
zero_points_or_biases_buffer: &self.biases_buffer,
output_buffer,
- batch: batch_size as i32,
- input_dim: self.model_dim as i32,
- output_dim: self.vocab_size as i32,
- quantization_type: QuantizationType::Mlx,
+ batch: batch_size,
+ input_dim: self.model_dim,
+ output_dim: self.vocab_size,
+ quantization_type: QuantizedMatmulType::Mlx,
};
self.kernel.encode(encoder, args).expect("Failed to encode quantized embedding readout kernel");
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/common.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/common.h
new file mode 100644
index 000000000..2535a2ea4
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/common.h
@@ -0,0 +1,4 @@
+#pragma once
+
+// Shared quantized matmul helper implementations.
+#include "quant_matmul.metal"
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index 876dc6407..f22db7374 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -1,30 +1,31 @@
use std::collections::HashMap;
-use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};
+use metal::{MTLBuffer, MTLComputeCommandEncoder};
use objc2::{rc::Retained, runtime::ProtocolObject};
use crate::{
DataType,
backends::{
- common::kernel::matmul::{
- QuantizedMatmulArguments as GenericQuantizedMatmulArguments, QuantizedMatmulConfiguration,
- QuantizedMatmulKernel as QuantizedMatmulKernelTrait, QuantizedMatmulType,
+ common::kernel::{
+ QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
+ QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,
+ matmul::{
+ QuantizedMatmulArguments as GenericQuantizedMatmulArguments, QuantizedMatmulConfiguration,
+ QuantizedMatmulKernel as QuantizedMatmulKernelTrait, QuantizedMatmulType,
+ },
},
metal::{
- FunctionConstantValuesSetValue, Metal, MetalContext, MetalError, metal_extensions::ComputeEncoderSetValue,
+ Metal, MetalContext, MetalError,
+ kernel::dsl::{
+ QuantizedMatmulQmmMetalKernel, QuantizedMatmulQmmTransposed64x64MetalKernel,
+ QuantizedMatmulQmmTransposedMetalKernel, QuantizedMatmulQmvFastMetalKernel,
+ QuantizedMatmulQmvMetalKernel, QuantizedMatmulQvmMetalKernel,
+ },
},
},
config::QuantizationMode,
};
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum QuantizationType {
- /// GroupQuantized style: uses zero-points
- ZeroPoint,
- /// MLX style: uses pre-computed biases
- Mlx,
-}
-
#[derive(Debug, thiserror::Error)]
pub enum QuantizedMatmulError {
#[error("Metal error: {0}")]
@@ -33,205 +34,237 @@ pub enum QuantizedMatmulError {
UnsupportedDataType(DataType),
#[error("Unsupported group size: {0}")]
UnsupportedGroupSize(usize),
- #[error("Invalid dimensions: M={m}, N={n}, K={k}")]
- InvalidDimensions {
- m: usize,
- n: usize,
- k: usize,
- },
#[error("Unsupported bits: {0}")]
UnsupportedBits(usize),
+ #[error("Value `{name}` does not fit i32: {value}")]
+ ValueOutOfRange {
+ name: &'static str,
+ value: usize,
+ },
+ #[error("Quantization type mismatch: kernel={kernel:?}, args={args:?}")]
+ QuantizationTypeMismatch {
+ kernel: QuantizedMatmulType,
+ args: QuantizedMatmulType,
+ },
+ #[error("Missing kernel for key: {0}")]
+ MissingKernel(&'static str),
}
pub struct QuantizedMatmulKernel {
- pipelines: HashMap<KernelKind, (Retained<ProtocolObject<dyn MTLComputePipelineState>>, u64, u64)>,
+ kernels: HashMap<KernelKey, EncodableVariant>,
+ matrix_vector_key: KernelKey,
+ matrix_matrix_key: KernelKey,
output_dim: usize,
- weights_transposed: bool,
+ quantization_type: QuantizedMatmulType,
}
-#[derive(Debug, Clone, Copy)]
-pub struct QuantizedMatmulArguments<'a> {
- pub a_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- /// Byte offset into `a_buffer` (used for slicing the batch dimension).
- pub a_offset: u64,
- pub b_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub scales_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub zero_points_or_biases_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>,
- pub batch: i32,
- pub input_dim: i32,
- pub output_dim: i32,
- pub quantization_type: QuantizationType,
+enum EncodableVariant {
+ Qmv(QuantizedMatmulQmvMetalKernel),
+ QmvFast(QuantizedMatmulQmvFastMetalKernel),
+ Qvm(QuantizedMatmulQvmMetalKernel),
+ Qmm(QuantizedMatmulQmmMetalKernel),
+ QmmTransposed(QuantizedMatmulQmmTransposedMetalKernel),
+ QmmTransposed64x64(QuantizedMatmulQmmTransposed64x64MetalKernel),
}
-fn dtype_suffix(data_type: DataType) -> Option<&'static str> {
- match data_type {
- DataType::F16 => Some("f16"),
- DataType::BF16 => Some("bf16"),
- DataType::F32 => Some("f32"),
- _ => None,
- }
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+enum KernelKey {
+ MatrixVector(MatrixVectorFamily),
+ MatrixMatrix(MatrixMatrixFamily),
}
-fn base_qmm_kernel_name(
- type_suffix: &str,
- group_size: usize,
- transpose_infix: &str,
- bits: usize,
-) -> Result<String, QuantizedMatmulError> {
- if bits != 4 && bits != 8 {
- return Err(QuantizedMatmulError::UnsupportedBits(bits));
- }
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+enum MatrixVectorFamily {
+ Qmv,
+ QmvFast,
+ Qvm,
+}
- let kernel_name = match (type_suffix, group_size) {
- ("f16", 32) => format!("qmm{}_f16_g32_b{}", transpose_infix, bits),
- ("f16", 64) => format!("qmm{}_f16_g64_b{}", transpose_infix, bits),
- ("f16", 128) => format!("qmm{}_f16_g128_b{}", transpose_infix, bits),
- ("bf16", 32) => format!("qmm{}_bf16_g32_b{}", transpose_infix, bits),
- ("bf16", 64) => format!("qmm{}_bf16_g64_b{}", transpose_infix, bits),
- ("bf16", 128) => format!("qmm{}_bf16_g128_b{}", transpose_infix, bits),
- ("f32", 32) => format!("qmm{}_f32_g32_b{}", transpose_infix, bits),
- ("f32", 64) => format!("qmm{}_f32_g64_b{}", transpose_infix, bits),
- ("f32", 128) => format!("qmm{}_f32_g128_b{}", transpose_infix, bits),
- _ => {
- return Err(QuantizedMatmulError::UnsupportedGroupSize(group_size));
- },
- };
- Ok(kernel_name)
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+enum MatrixMatrixFamily {
+ QmmAlignedK,
+ QmmUnalignedK,
+ QmmTransposedAlignedN,
+ QmmTransposedUnalignedN,
+ QmmTransposed64x64,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RuntimeVariant {
+ MatrixVector,
+ MatrixMatrix,
}
impl QuantizedMatmulKernel {
pub fn new(
- mtl_context: &MetalContext,
- data_type: DataType,
- group_size: usize,
- input_dim: usize,
- output_dim: usize,
- mode: QuantizationMode,
- quantization_type: QuantizationType,
- weights_transposed: bool,
+ context: &MetalContext,
+ configuration: QuantizedMatmulConfiguration,
) -> Result<Self, QuantizedMatmulError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(QuantizedMatmulError::UnsupportedDataType(data_type));
- }
-
- let function_constants = MTLFunctionConstantValues::new();
- let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);
- function_constants.set_value(&use_mlx_quant, 40);
-
- let mut pipelines = HashMap::new();
-
- let kernel_name_mv =
- select_matrix_vector_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;
-
- let cache_key_mv = format!("{}_mlx_{}", kernel_name_mv, use_mlx_quant);
- let pipeline_mv = mtl_context
- .compute_pipeline_state_cached(&cache_key_mv, &kernel_name_mv, Some(&function_constants))
- .map_err(QuantizedMatmulError::MetalError)?;
- pipelines.insert(KernelKind::MatrixVector, (pipeline_mv, 32, 32));
-
- let kernel_name_mm =
- select_qmm_kernel_name(data_type, group_size, weights_transposed, output_dim, input_dim, mode)?;
-
- let cache_key_mm = format!("{}_mlx_{}", kernel_name_mm, use_mlx_quant);
- let pipeline_mm = mtl_context
- .compute_pipeline_state_cached(&cache_key_mm, &kernel_name_mm, Some(&function_constants))
- .map_err(QuantizedMatmulError::MetalError)?;
-
- let (bm, bn) = if kernel_name_mm.contains("_64x64") {
- (64, 64)
- } else if kernel_name_mm.contains("_64x128") {
- (64, 128)
- } else if kernel_name_mm.contains("_128x64") {
- (128, 64)
- } else {
- (32, 32)
- };
- pipelines.insert(KernelKind::MatrixMatrix, (pipeline_mm, bm, bn));
+ validate_configuration(&configuration)?;
+
+ let bits = quant_bits(configuration.mode)?;
+ let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);
+
+ let matrix_vector_family = select_matrix_vector_family(&configuration);
+ let matrix_matrix_family = select_matrix_matrix_family(&configuration, bits);
+ let matrix_vector_key = KernelKey::MatrixVector(matrix_vector_family);
+ let matrix_matrix_key = KernelKey::MatrixMatrix(matrix_matrix_family);
+
+ let mut kernels = HashMap::new();
+ kernels.insert(
+ matrix_vector_key,
+ create_matrix_vector_kernel(
+ context,
+ configuration.data_type,
+ configuration.group_size,
+ bits,
+ use_mlx_quant,
+ matrix_vector_family,
+ )?,
+ );
+ kernels.insert(
+ matrix_matrix_key,
+ create_matrix_matrix_kernel(
+ context,
+ configuration.data_type,
+ configuration.group_size,
+ bits,
+ use_mlx_quant,
+ matrix_matrix_family,
+ )?,
+ );
Ok(Self {
- pipelines,
- output_dim,
- weights_transposed,
+ kernels,
+ matrix_vector_key,
+ matrix_matrix_key,
+ output_dim: configuration.output_dim,
+ quantization_type: configuration.quantization_type,
})
}
- #[allow(dead_code)]
- fn kernel_name_for_config(data_type: DataType) -> String {
- let type_suffix = match data_type {
- DataType::F16 => "f16",
- DataType::BF16 => "bf16",
- DataType::F32 => "f32",
- _ => unreachable!(),
- };
-
- format!("qmm_t_{}_gs_64_b_4_batch_0", type_suffix)
- }
-
pub fn encode(
&self,
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: QuantizedMatmulArguments,
+ arguments: GenericQuantizedMatmulArguments<Metal>,
) -> Result<(), QuantizedMatmulError> {
- let variant = self.select_variant(args.batch as usize);
- let (pipeline, bm, bn) =
- self.pipelines.get(&variant).ok_or_else(|| QuantizedMatmulError::InvalidDimensions {
- m: args.batch as usize,
- n: args.output_dim as usize,
- k: args.input_dim as usize,
- })?;
-
- encoder.set_compute_pipeline_state(pipeline);
-
- // Set buffers
- encoder.set_buffer(Some(args.b_buffer), 0, 0);
- encoder.set_buffer(Some(args.scales_buffer), 0, 1);
- encoder.set_buffer(Some(args.zero_points_or_biases_buffer), 0, 2);
- encoder.set_buffer(Some(args.a_buffer), args.a_offset as usize, 3);
- encoder.set_buffer(Some(args.output_buffer), 0, 4);
-
- let k: i32 = args.input_dim;
- let m = args.batch;
- let n = args.output_dim;
-
- encoder.set_value(&k, 5);
- encoder.set_value(&n, 6);
- encoder.set_value(&m, 7);
-
- match variant {
- KernelKind::MatrixVector => {
- let bk = 32;
- let bn = if self.weights_transposed {
- 8
- } else {
- 64
- };
- let n_tgp_y = (n + bn - 1) / bn;
- let threadgroups = MTLSize::new(m as usize, n_tgp_y as usize, 1);
- let threads_per_threadgroup = MTLSize::new(bk as usize, 2, 1);
- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);
+ if arguments.quantization_type != self.quantization_type {
+ return Err(QuantizedMatmulError::QuantizationTypeMismatch {
+ kernel: self.quantization_type,
+ args: arguments.quantization_type,
+ });
+ }
+
+ let key = match self.select_runtime_variant(arguments.batch) {
+ RuntimeVariant::MatrixVector => self.matrix_vector_key,
+ RuntimeVariant::MatrixMatrix => self.matrix_matrix_key,
+ };
+
+ let kernel = self.kernels.get(&key).ok_or(QuantizedMatmulError::MissingKernel(kernel_key_name(key)))?;
+ let k = to_i32("input_dim", arguments.input_dim)?;
+ let n = to_i32("output_dim", arguments.output_dim)?;
+ let m = to_i32("batch", arguments.batch)?;
+ let (zero_points, biases) = quant_buffers(arguments.zero_points_or_biases_buffer, self.quantization_type);
+ let a_with_offset = (arguments.a_buffer, arguments.a_offset);
+
+ match kernel {
+ EncodableVariant::Qmv(kernel) => {
+ kernel.encode(
+ arguments.b_buffer,
+ arguments.scales_buffer,
+ zero_points,
+ biases,
+ a_with_offset,
+ arguments.output_buffer,
+ k,
+ n,
+ m,
+ encoder,
+ );
},
- KernelKind::MatrixMatrix => {
- let wm = 2;
- let wn = 2;
- let threads_per_threadgroup = MTLSize::new(32, wn as usize, wm as usize);
- let threadgroups =
- MTLSize::new(((n as u64 + bn - 1) / bn) as usize, ((m as u64 + bm - 1) / bm) as usize, 1);
- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);
+ EncodableVariant::QmvFast(kernel) => {
+ kernel.encode(
+ arguments.b_buffer,
+ arguments.scales_buffer,
+ zero_points,
+ biases,
+ a_with_offset,
+ arguments.output_buffer,
+ k,
+ n,
+ m,
+ encoder,
+ );
+ },
+ EncodableVariant::Qvm(kernel) => {
+ kernel.encode(
+ arguments.b_buffer,
+ arguments.scales_buffer,
+ zero_points,
+ biases,
+ a_with_offset,
+ arguments.output_buffer,
+ k,
+ n,
+ m,
+ encoder,
+ );
+ },
+ EncodableVariant::Qmm(kernel) => {
+ kernel.encode(
+ arguments.b_buffer,
+ arguments.scales_buffer,
+ zero_points,
+ biases,
+ a_with_offset,
+ arguments.output_buffer,
+ k,
+ n,
+ m,
+ encoder,
+ );
+ },
+ EncodableVariant::QmmTransposed(kernel) => {
+ kernel.encode(
+ arguments.b_buffer,
+ arguments.scales_buffer,
+ zero_points,
+ biases,
+ a_with_offset,
+ arguments.output_buffer,
+ k,
+ n,
+ m,
+ encoder,
+ );
+ },
+ EncodableVariant::QmmTransposed64x64(kernel) => {
+ kernel.encode(
+ arguments.b_buffer,
+ arguments.scales_buffer,
+ zero_points,
+ biases,
+ a_with_offset,
+ arguments.output_buffer,
+ k,
+ n,
+ m,
+ encoder,
+ );
},
}
Ok(())
}
- fn select_variant(
+ fn select_runtime_variant(
&self,
batch: usize,
- ) -> KernelKind {
+ ) -> RuntimeVariant {
if batch < 32 || self.output_dim == 1 {
- KernelKind::MatrixVector
+ RuntimeVariant::MatrixVector
} else {
- KernelKind::MatrixMatrix
+ RuntimeVariant::MatrixMatrix
}
}
}
@@ -243,22 +276,7 @@ impl QuantizedMatmulKernelTrait for QuantizedMatmulKernel {
context: &MetalContext,
configuration: QuantizedMatmulConfiguration,
) -> Result<Self, MetalError> {
- let quantization_type = match configuration.quantization_type {
- QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,
- QuantizedMatmulType::Mlx => QuantizationType::Mlx,
- };
-
- QuantizedMatmulKernel::new(
- context,
- configuration.data_type,
- configuration.group_size,
- configuration.input_dim,
- configuration.output_dim,
- configuration.mode,
- quantization_type,
- configuration.weights_transposed,
- )
- .map_err(|error| MetalError::Generic(format!("{:?}", error)))
+ QuantizedMatmulKernel::new(context, configuration).map_err(|error| MetalError::Generic(format!("{error:?}")))
}
fn encode(
@@ -266,270 +284,212 @@ impl QuantizedMatmulKernelTrait for QuantizedMatmulKernel {
encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
arguments: GenericQuantizedMatmulArguments<Metal>,
) {
- let quantization_type = match arguments.quantization_type {
- QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,
- QuantizedMatmulType::Mlx => QuantizationType::Mlx,
- };
-
- let backend_arguments = QuantizedMatmulArguments {
- a_buffer: arguments.a_buffer,
- a_offset: arguments.a_offset as u64,
- b_buffer: arguments.b_buffer,
- scales_buffer: arguments.scales_buffer,
- zero_points_or_biases_buffer: arguments.zero_points_or_biases_buffer,
- output_buffer: arguments.output_buffer,
- batch: arguments.batch as i32,
- input_dim: arguments.input_dim as i32,
- output_dim: arguments.output_dim as i32,
- quantization_type,
- };
-
- QuantizedMatmulKernel::encode(self, encoder, backend_arguments).expect("Failed to encode quantized matmul");
+ QuantizedMatmulKernel::encode(self, encoder, arguments).expect("Failed to encode quantized matmul");
}
}
-fn select_matrix_vector_kernel_name(
+fn create_matrix_vector_kernel(
+ context: &MetalContext,
data_type: DataType,
group_size: usize,
- weights_transposed: bool,
- output_dim: usize,
- input_dim: usize,
- mode: QuantizationMode,
-) -> Result<String, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
- let bits = match mode {
- QuantizationMode::UInt4 => 4,
- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
+ bits: usize,
+ use_mlx_quant: bool,
+ family: MatrixVectorFamily,
+) -> Result<EncodableVariant, QuantizedMatmulError> {
+ let group_size = to_i32("group_size", group_size)?;
+ let bits = to_i32("bits", bits)?;
+ let use_zero_points = !use_mlx_quant;
+
+ let kernel = match family {
+ MatrixVectorFamily::Qmv => EncodableVariant::Qmv(QuantizedMatmulQmvMetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )?),
+ MatrixVectorFamily::QmvFast => EncodableVariant::QmvFast(QuantizedMatmulQmvFastMetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )?),
+ MatrixVectorFamily::Qvm => EncodableVariant::Qvm(QuantizedMatmulQvmMetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )?),
};
- if weights_transposed {
- let mut name = format!("qmv_{}_g{}_b{}", type_suffix, group_size, bits);
- if output_dim % 8 == 0 && input_dim % 512 == 0 {
- name.push_str("_fast");
- }
- return Ok(name);
- }
-
- Ok(format!("qvm_{}_g{}_b{}", type_suffix, group_size, bits))
+ Ok(kernel)
}
-fn select_qmm_kernel_name(
+fn create_matrix_matrix_kernel(
+ context: &MetalContext,
data_type: DataType,
group_size: usize,
- weights_transposed: bool,
- output_dim: usize,
- input_dim: usize,
- mode: QuantizationMode,
-) -> Result<String, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
- let bits = match mode {
- QuantizationMode::UInt4 => 4,
- QuantizationMode::Int8 | QuantizationMode::UInt8 => 8,
+ bits: usize,
+ use_mlx_quant: bool,
+ family: MatrixMatrixFamily,
+) -> Result<EncodableVariant, QuantizedMatmulError> {
+ let group_size = to_i32("group_size", group_size)?;
+ let bits = to_i32("bits", bits)?;
+ let use_zero_points = !use_mlx_quant;
+
+ let kernel = match family {
+ MatrixMatrixFamily::QmmAlignedK => EncodableVariant::Qmm(QuantizedMatmulQmmMetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ true,
+ )?),
+ MatrixMatrixFamily::QmmUnalignedK => EncodableVariant::Qmm(QuantizedMatmulQmmMetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ false,
+ )?),
+ MatrixMatrixFamily::QmmTransposedAlignedN => {
+ EncodableVariant::QmmTransposed(QuantizedMatmulQmmTransposedMetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ true,
+ )?)
+ },
+ MatrixMatrixFamily::QmmTransposedUnalignedN => {
+ EncodableVariant::QmmTransposed(QuantizedMatmulQmmTransposedMetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ false,
+ )?)
+ },
+ MatrixMatrixFamily::QmmTransposed64x64 => {
+ EncodableVariant::QmmTransposed64x64(QuantizedMatmulQmmTransposed64x64MetalKernel::new(
+ context,
+ data_type,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )?)
+ },
};
- let transpose_infix = if weights_transposed {
- "_transposed"
- } else {
- ""
- };
- let mut kernel_name = base_qmm_kernel_name(type_suffix, group_size, transpose_infix, bits)?;
- if weights_transposed {
- if output_dim % 32 != 0 {
- kernel_name.push_str("_unaligned");
- } else if type_suffix == "bf16" && (group_size == 128 || group_size == 64) && (bits == 4 || bits == 8) {
- kernel_name.push_str("_64x64");
- }
- } else if input_dim % 32 == 0 {
- kernel_name.push_str("_alignedk");
- }
- Ok(kernel_name)
+ Ok(kernel)
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-enum KernelKind {
- MatrixMatrix,
- MatrixVector,
-}
+fn validate_configuration(configuration: &QuantizedMatmulConfiguration) -> Result<(), QuantizedMatmulError> {
+ if !matches!(configuration.data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(QuantizedMatmulError::UnsupportedDataType(configuration.data_type));
+ }
-/// Arguments for MLP fused quantized GEMV
-#[derive(Debug)]
-pub struct MlpFusedQmvArguments<'a> {
- /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated, quantized)
- pub weights: &'a ProtocolObject<dyn MTLBuffer>,
- /// Scales buffer
- pub scales: &'a ProtocolObject<dyn MTLBuffer>,
- /// Zero points or biases buffer (depends on quantization type)
- pub zero_points_or_biases: &'a ProtocolObject<dyn MTLBuffer>,
- /// Input vector [input_dim]
- pub input: &'a ProtocolObject<dyn MTLBuffer>,
- /// Input byte offset
- pub input_offset: u64,
- /// Output vector [hidden_dim]
- pub output: &'a ProtocolObject<dyn MTLBuffer>,
- /// Input dimension (K)
- pub input_dim: i32,
- /// Hidden dimension (output size, half of weight rows)
- pub hidden_dim: i32,
- /// Batch count
- pub batch_count: i32,
-}
+ if !matches!(configuration.group_size, 32 | 64 | 128) {
+ return Err(QuantizedMatmulError::UnsupportedGroupSize(configuration.group_size));
+ }
-/// MLP Fused Quantized GEMV Kernel
-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
-pub struct MlpFusedQmvKernel {
- pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+ let _ = quant_bits(configuration.mode)?;
+ Ok(())
}
-impl MlpFusedQmvKernel {
- pub fn new(
- context: &MetalContext,
- data_type: DataType,
- group_size: usize,
- mode: QuantizationMode,
- quantization_type: QuantizationType,
- ) -> Result<Self, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
-
- let bits = match mode {
- QuantizationMode::UInt4 => 4,
- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
- };
-
- if bits != 4 {
- return Err(QuantizedMatmulError::UnsupportedBits(bits));
+fn select_matrix_vector_family(configuration: &QuantizedMatmulConfiguration) -> MatrixVectorFamily {
+ if configuration.weights_transposed {
+ if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {
+ MatrixVectorFamily::QmvFast
+ } else {
+ MatrixVectorFamily::Qmv
}
-
- let kernel_name = format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
-
- let function_constants = MTLFunctionConstantValues::new();
- let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);
- function_constants.set_value(&use_mlx_quant, 40);
-
- let pipeline = context
- .compute_pipeline_state(&kernel_name, Some(&function_constants))
- .map_err(QuantizedMatmulError::MetalError)?;
-
- Ok(Self {
- pipeline,
- })
+ } else {
+ MatrixVectorFamily::Qvm
}
+}
- pub fn encode(
- &self,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: &MlpFusedQmvArguments,
- ) -> Result<(), QuantizedMatmulError> {
- encoder.set_compute_pipeline_state(&self.pipeline);
-
- encoder.set_buffer(Some(args.weights), 0, 0);
- encoder.set_buffer(Some(args.scales), 0, 1);
- encoder.set_buffer(Some(args.zero_points_or_biases), 0, 2);
- encoder.set_buffer(Some(args.input), args.input_offset as usize, 3);
- encoder.set_buffer(Some(args.output), 0, 4);
-
- encoder.set_value(&args.input_dim, 5);
- encoder.set_value(&args.hidden_dim, 6);
-
- // Dispatch: one threadgroup per 8 output rows (num_simdgroups * results_per_simdgroup)
- let rows_per_threadgroup = 8;
- let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1) / rows_per_threadgroup) as u64;
- let threadgroups = MTLSize::new(args.batch_count.max(1) as usize, n_tgp_y as usize, 1);
- let threads_per_threadgroup = MTLSize::new(32, 2, 1); // 2 simdgroups, 32 threads each
-
- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);
- Ok(())
+fn select_matrix_matrix_family(
+ configuration: &QuantizedMatmulConfiguration,
+ bits: usize,
+) -> MatrixMatrixFamily {
+ if configuration.weights_transposed {
+ let aligned_n = configuration.output_dim % 32 == 0;
+ let use_64x64 = aligned_n
+ && configuration.data_type == DataType::BF16
+ && matches!(configuration.group_size, 64 | 128)
+ && matches!(bits, 4 | 8);
+ if use_64x64 {
+ MatrixMatrixFamily::QmmTransposed64x64
+ } else if aligned_n {
+ MatrixMatrixFamily::QmmTransposedAlignedN
+ } else {
+ MatrixMatrixFamily::QmmTransposedUnalignedN
+ }
+ } else if configuration.input_dim % 32 == 0 {
+ MatrixMatrixFamily::QmmAlignedK
+ } else {
+ MatrixMatrixFamily::QmmUnalignedK
}
}
-/// Arguments for MLP fused quantized GEMM (prefill)
-#[derive(Debug)]
-pub struct MlpFusedQmmArguments<'a> {
- /// Weight matrix [K, 2*hidden_dim] quantized (up and gate concatenated)
- pub weights: &'a ProtocolObject<dyn MTLBuffer>,
- /// Scales buffer
- pub scales: &'a ProtocolObject<dyn MTLBuffer>,
- /// Zero points or biases buffer (depends on quantization type)
- pub zero_points_or_biases: &'a ProtocolObject<dyn MTLBuffer>,
- /// Input activations [M, K]
- pub input: &'a ProtocolObject<dyn MTLBuffer>,
- /// Input byte offset
- pub input_offset: u64,
- /// Output [M, hidden_dim]
- pub output: &'a ProtocolObject<dyn MTLBuffer>,
- /// Batch size (M)
- pub batch: i32,
- /// Input dimension (K)
- pub input_dim: i32,
- /// Hidden dimension (output size, half of weight columns)
- pub hidden_dim: i32,
+fn quant_bits(mode: QuantizationMode) -> Result<usize, QuantizedMatmulError> {
+ let bits = match mode {
+ QuantizationMode::UInt4 => 4,
+ QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
+ };
+ if matches!(bits, 4 | 8) {
+ Ok(bits)
+ } else {
+ Err(QuantizedMatmulError::UnsupportedBits(bits))
+ }
}
-/// MLP Fused Quantized GEMM Kernel for prefill path
-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
-pub struct MlpFusedQmmKernel {
- pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
+fn to_i32(
+ name: &'static str,
+ value: usize,
+) -> Result<i32, QuantizedMatmulError> {
+ i32::try_from(value).map_err(|_| QuantizedMatmulError::ValueOutOfRange {
+ name,
+ value,
+ })
}
-impl MlpFusedQmmKernel {
- pub fn new(
- context: &MetalContext,
- data_type: DataType,
- group_size: usize,
- mode: QuantizationMode,
- quantization_type: QuantizationType,
- ) -> Result<Self, QuantizedMatmulError> {
- let type_suffix = dtype_suffix(data_type).ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
-
- let bits = match mode {
- QuantizationMode::UInt4 => 4,
- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
- };
-
- if bits != 4 {
- return Err(QuantizedMatmulError::UnsupportedBits(bits));
- }
-
- let kernel_name = format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
-
- let function_constants = MTLFunctionConstantValues::new();
- let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);
- function_constants.set_value(&use_mlx_quant, 40);
-
- let pipeline = context
- .compute_pipeline_state(&kernel_name, Some(&function_constants))
- .map_err(QuantizedMatmulError::MetalError)?;
-
- Ok(Self {
- pipeline,
- })
+fn quant_buffers<'a>(
+ buffer: &'a Retained<ProtocolObject<dyn MTLBuffer>>,
+ quantization_type: QuantizedMatmulType,
+) -> (Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>, Option<&'a Retained<ProtocolObject<dyn MTLBuffer>>>) {
+ match quantization_type {
+ QuantizedMatmulType::ZeroPoint => (Some(buffer), None),
+ QuantizedMatmulType::Mlx => (None, Some(buffer)),
}
+}
- pub fn encode(
- &self,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- args: &MlpFusedQmmArguments,
- ) -> Result<(), QuantizedMatmulError> {
- encoder.set_compute_pipeline_state(&self.pipeline);
-
- encoder.set_buffer(Some(args.weights), 0, 0);
- encoder.set_buffer(Some(args.scales), 0, 1);
- encoder.set_buffer(Some(args.zero_points_or_biases), 0, 2);
- encoder.set_buffer(Some(args.input), args.input_offset as usize, 3);
- encoder.set_buffer(Some(args.output), 0, 4);
-
- encoder.set_value(&args.input_dim, 5);
- encoder.set_value(&args.hidden_dim, 6);
- encoder.set_value(&args.batch, 7);
-
- // Dispatch: BM=32, BN=32
- let bm = 32;
- let bn = 32;
- let tiles_m = ((args.batch + bm - 1) / bm) as u64;
- let tiles_n = ((args.hidden_dim + bn - 1) / bn) as u64;
- let threadgroups = MTLSize::new(tiles_n as usize, tiles_m as usize, 1);
- let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads
-
- encoder.dispatch_threadgroups(threadgroups, threads_per_threadgroup);
- Ok(())
+fn kernel_key_name(key: KernelKey) -> &'static str {
+ match key {
+ KernelKey::MatrixVector(MatrixVectorFamily::Qmv) => "matrix_vector_qmv",
+ KernelKey::MatrixVector(MatrixVectorFamily::QmvFast) => "matrix_vector_qmv_fast",
+ KernelKey::MatrixVector(MatrixVectorFamily::Qvm) => "matrix_vector_qvm",
+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmAlignedK) => "matrix_matrix_qmm_aligned_k",
+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmUnalignedK) => "matrix_matrix_qmm_unaligned_k",
+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedAlignedN) => "matrix_matrix_qmm_transposed_aligned_n",
+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposedUnalignedN) => {
+ "matrix_matrix_qmm_transposed_unaligned_n"
+ },
+ KernelKey::MatrixMatrix(MatrixMatrixFamily::QmmTransposed64x64) => "matrix_matrix_qmm_transposed_64x64",
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
new file mode 100644
index 000000000..33232756b
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
@@ -0,0 +1,113 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+#include "common.h"
+
+template <typename T, int group_size, int bits>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(group_size, 32, 64, 128)
+VARIANTS(bits, 4, 8)
+KERNEL(QuantizedMatmulQmm)(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points OPTIONAL(use_zero_points),
+ const device T* biases OPTIONAL(use_mlx_quant),
+ const device T* x,
+ device T* y,
+ const constant int& k,
+ const constant int& n,
+ const constant int& m,
+ threadgroup T Xs[32 * (32 + 16 / sizeof(T))],
+ threadgroup T Ws[32 * (32 + 16 / sizeof(T))],
+ const bool use_zero_points SPECIALIZE,
+ const bool use_mlx_quant SPECIALIZE,
+ const bool aligned_k SPECIALIZE,
+ const uint tgid_x GROUPS((n + 32 - 1) / 32),
+ const uint tgid_y GROUPS((m + 32 - 1) / 32),
+ const uint tgid_z GROUPS(1),
+ const uint tid_x THREADS(32),
+ const uint tid_y THREADS(2),
+ const uint tid_z THREADS(2)
+) {
+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
+ const uint simd_gid = tid_z * 2 + tid_y;
+ const uint simd_lid = tid_x;
+
+ if (use_mlx_quant) {
+ if (aligned_k) {
+ qmm_impl<T, group_size, bits, true, 32, 32, 32, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmm_impl<T, group_size, bits, false, 32, 32, 32, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ }
+ } else {
+ if (aligned_k) {
+ qmm_impl<T, group_size, bits, true, 32, 32, 32, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmm_impl<T, group_size, bits, false, 32, 32, 32, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
new file mode 100644
index 000000000..2afb794fa
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -0,0 +1,113 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+#include "common.h"
+
+template <typename T, int group_size, int bits>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(group_size, 32, 64, 128)
+VARIANTS(bits, 4, 8)
+KERNEL(QuantizedMatmulQmmTransposed)(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points OPTIONAL(use_zero_points),
+ const device T* biases OPTIONAL(use_mlx_quant),
+ const device T* x,
+ device T* y,
+ const constant int& k,
+ const constant int& n,
+ const constant int& m,
+ threadgroup T Xs[32 * (32 + 16 / sizeof(T))],
+ threadgroup T Ws[32 * (32 + 16 / sizeof(T))],
+ const bool use_zero_points SPECIALIZE,
+ const bool use_mlx_quant SPECIALIZE,
+ const bool aligned_n SPECIALIZE,
+ const uint tgid_x GROUPS((n + 32 - 1) / 32),
+ const uint tgid_y GROUPS((m + 32 - 1) / 32),
+ const uint tgid_z GROUPS(1),
+ const uint tid_x THREADS(32),
+ const uint tid_y THREADS(2),
+ const uint tid_z THREADS(2)
+) {
+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
+ const uint simd_gid = tid_z * 2 + tid_y;
+ const uint simd_lid = tid_x;
+
+ if (use_mlx_quant) {
+ if (aligned_n) {
+ qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ }
+ } else {
+ if (aligned_n) {
+ qmm_transposed_impl<T, group_size, bits, true, 32, 32, 32, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmm_transposed_impl<T, group_size, bits, false, 32, 32, 32, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
new file mode 100644
index 000000000..0f7323e35
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
@@ -0,0 +1,72 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+#include "common.h"
+
+template <typename T, int group_size, int bits>
+VARIANTS(T, bfloat)
+VARIANTS(group_size, 64, 128)
+VARIANTS(bits, 4, 8)
+KERNEL(QuantizedMatmulQmmTransposed64x64)(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points OPTIONAL(use_zero_points),
+ const device T* biases OPTIONAL(use_mlx_quant),
+ const device T* x,
+ device T* y,
+ const constant int& k,
+ const constant int& n,
+ const constant int& m,
+ threadgroup T Xs[64 * (64 + 16 / sizeof(T))],
+ threadgroup T Ws[64 * (64 + 16 / sizeof(T))],
+ const bool use_zero_points SPECIALIZE,
+ const bool use_mlx_quant SPECIALIZE,
+ const uint tgid_x GROUPS((n + 64 - 1) / 64),
+ const uint tgid_y GROUPS((m + 64 - 1) / 64),
+ const uint tgid_z GROUPS(1),
+ const uint tid_x THREADS(32),
+ const uint tid_y THREADS(2),
+ const uint tid_z THREADS(2)
+) {
+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
+ const uint lid = tid_z * 64 + tid_y * 32 + tid_x;
+ const uint simd_gid = tid_z * 2 + tid_y;
+ const uint simd_lid = tid_x;
+
+ if (use_mlx_quant) {
+ qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmm_transposed_impl<T, group_size, bits, true, 64, 64, 64, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ k,
+ n,
+ m,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
new file mode 100644
index 000000000..ac62ebb48
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
@@ -0,0 +1,60 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+#include "common.h"
+
+template <typename T, int group_size, int bits>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(group_size, 32, 64, 128)
+VARIANTS(bits, 4, 8)
+KERNEL(QuantizedMatmulQmv)(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points OPTIONAL(use_zero_points),
+ const device T* biases OPTIONAL(use_mlx_quant),
+ const device T* x,
+ device T* y,
+ const constant int& k,
+ const constant int& n,
+ const constant int& m,
+ const bool use_zero_points SPECIALIZE,
+ const bool use_mlx_quant SPECIALIZE,
+ const uint tgid_x GROUPS(m),
+ const uint tgid_y GROUPS((n + 8 - 1) / 8),
+ const uint tgid_z GROUPS(1),
+ const uint tid_x THREADS(32),
+ const uint tid_y THREADS(2)
+) {
+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
+ const uint simd_gid = tid_y;
+ const uint simd_lid = tid_x;
+
+ if (use_mlx_quant) {
+ qmv_impl<T, group_size, bits, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ k,
+ n,
+ tid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmv_impl<T, group_size, bits, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ k,
+ n,
+ tid,
+ simd_gid,
+ simd_lid
+ );
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
new file mode 100644
index 000000000..c911b1b3e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -0,0 +1,60 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+#include "common.h"
+
+template <typename T, int group_size, int bits>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(group_size, 32, 64, 128)
+VARIANTS(bits, 4, 8)
+KERNEL(QuantizedMatmulQmvFast)(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points OPTIONAL(use_zero_points),
+ const device T* biases OPTIONAL(use_mlx_quant),
+ const device T* x,
+ device T* y,
+ const constant int& k,
+ const constant int& n,
+ const constant int& m,
+ const bool use_zero_points SPECIALIZE,
+ const bool use_mlx_quant SPECIALIZE,
+ const uint tgid_x GROUPS(m),
+ const uint tgid_y GROUPS((n + 8 - 1) / 8),
+ const uint tgid_z GROUPS(1),
+ const uint tid_x THREADS(32),
+ const uint tid_y THREADS(2)
+) {
+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
+ const uint simd_gid = tid_y;
+ const uint simd_lid = tid_x;
+
+ if (use_mlx_quant) {
+ qmv_fast_impl<T, group_size, bits, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ k,
+ n,
+ tid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qmv_fast_impl<T, group_size, bits, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ k,
+ n,
+ tid,
+ simd_gid,
+ simd_lid
+ );
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index 9e83f09e3..c2408b656 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -4,12 +4,6 @@
using namespace metal;
#include "mma.h"
-#include "../common/mlp_epilogue.h"
-
-// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style
-// (zero-points)
-constant bool kUseMlxQuant [[function_constant(40)]];
-constant bool kUseZeroPoints = !kUseMlxQuant;
template <int bits, int wsize = 8>
inline constexpr short get_pack_factor() {
@@ -731,7 +725,8 @@ template <
const bool aligned_K = false,
const int BM = 32,
const int BK = 32,
- const int BN = 32>
+ const int BN = 32,
+ const bool use_mlx_quant = false>
void qmm_impl(
const device uint32_t* w,
const device T* scales,
@@ -783,7 +778,7 @@ void qmm_impl(
loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);
mma_t mma_op(simd_gid, simd_lid);
- if (kUseMlxQuant) {
+ if (use_mlx_quant) {
using loader_w_t = QuantizedBlockLoaderMlx<
T,
BK,
@@ -939,7 +934,8 @@ template <
const bool aligned_N,
const int BM = 32,
const int BK = 32,
- const int BN = 32>
+ const int BN = 32,
+ const bool use_mlx_quant = false>
void qmm_transposed_impl(
const device uint32_t* w,
const device T* scales,
@@ -989,7 +985,7 @@ void qmm_transposed_impl(
loader_x_t loader_x(x_block, K, Xs, simd_gid, simd_lid);
mma_t mma_op(simd_gid, simd_lid);
- if (kUseMlxQuant) {
+ if (use_mlx_quant) {
using loader_w_t = QuantizedBlockLoaderMlx<
T,
BN,
@@ -1317,7 +1313,7 @@ void qmv_impl_dispatch(
}
}
-template <typename T, int group_size, int bits>
+template <typename T, int group_size, int bits, bool use_mlx_quant>
void qmv_impl(
const device uint32_t* w,
const device T* scales,
@@ -1331,7 +1327,7 @@ void qmv_impl(
uint simd_gid [[simdgroup_index_in_threadgroup]],
uint simd_lid [[thread_index_in_simdgroup]]
) {
- if (kUseMlxQuant) {
+ if (use_mlx_quant) {
qmv_impl_dispatch<T, group_size, bits, true>(
w,
scales,
@@ -1362,7 +1358,7 @@ void qmv_impl(
}
}
-template <typename T, int group_size, int bits>
+template <typename T, int group_size, int bits, bool use_mlx_quant>
void qmv_fast_impl(
const device uint32_t* w,
const device T* scales,
@@ -1400,7 +1396,7 @@ void qmv_fast_impl(
const device uint8_t* zps = nullptr;
bool high_nibble = false;
- if (kUseMlxQuant) {
+ if (use_mlx_quant) {
biases += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
} else {
if (bits == 4) {
@@ -1433,7 +1429,7 @@ void qmv_fast_impl(
U s2 = static_cast<U>(scales[2 * in_vec_size_g]);
U s3 = static_cast<U>(scales[3 * in_vec_size_g]);
- if (kUseMlxQuant) {
+ if (use_mlx_quant) {
U b0 = static_cast<U>(biases[0]);
U b1 = static_cast<U>(biases[in_vec_size_g]);
U b2 = static_cast<U>(biases[2 * in_vec_size_g]);
@@ -1482,7 +1478,7 @@ void qmv_fast_impl(
ws += block_size * bytes_per_pack / pack_factor;
scales += block_size / group_size;
- if (kUseMlxQuant) {
+ if (use_mlx_quant) {
biases += block_size / group_size;
} else {
if (bits == 4) {
@@ -1674,7 +1670,7 @@ void qvm_impl_zeropoint(
);
}
-template <typename T, int group_size, int bits>
+template <typename T, int group_size, int bits, bool use_mlx_quant>
void qvm_impl(
const device uint32_t* w,
const device T* scales,
@@ -1688,7 +1684,7 @@ void qvm_impl(
uint simd_gid [[simdgroup_index_in_threadgroup]],
uint simd_lid [[thread_index_in_simdgroup]]
) {
- if (kUseMlxQuant) {
+ if (use_mlx_quant) {
qvm_impl_mlx<T, group_size, bits>(
w,
scales,
@@ -1716,3304 +1712,3 @@ void qvm_impl(
);
}
}
-
-template <
- typename T,
- const int group_size,
- const int bits,
- const bool aligned_K,
- const int BM = 32,
- const int BK = 32,
- const int BN = 32>
-[[kernel, max_total_threads_per_threadgroup(128)]] void qmm(
- const device uint32_t* w [[buffer(0)]],
- const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device T* x [[buffer(3)]],
- device T* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
-
- (void)lid;
-
- constexpr int BK_padded = (BK + 16 / sizeof(T));
- constexpr int BN_padded = (BN + 16 / sizeof(T));
-
- threadgroup T Xs[BM * BK_padded];
- threadgroup T Ws[BK * BN_padded];
-
- qmm_impl<T, group_size, bits, aligned_K, BM, BK, BN>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- K,
- N,
- M,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
-}
-
-template <
- typename T,
- const int group_size,
- const int bits,
- const bool aligned_N,
- const int BM = 32,
- const int BK = 32,
- const int BN = 32>
-[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_transposed(
- const device uint32_t* w [[buffer(0)]],
- const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device T* x [[buffer(3)]],
- device T* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
-
- (void)lid;
-
- constexpr int BK_padded = (BK + 16 / sizeof(T));
-
- threadgroup T Xs[BM * BK_padded];
- threadgroup T Ws[BN * BK_padded];
-
- qmm_transposed_impl<T, group_size, bits, aligned_N, BM, BK, BN>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- K,
- N,
- M,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
-}
-
-template <typename T, int group_size, int bits>
-[[kernel, max_total_threads_per_threadgroup(64)]] void qmv(
- const device uint32_t* w [[buffer(0)]],
- const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device T* x [[buffer(3)]],
- device T* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
-
- qmv_impl<T, group_size, bits>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- K,
- N,
- tid,
- simd_gid,
- simd_lid
- );
-}
-
-template <typename T, int group_size, int bits>
-[[kernel, max_total_threads_per_threadgroup(64)]] void qvm(
- const device uint32_t* w [[buffer(0)]],
- const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device T* x [[buffer(3)]],
- device T* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
-
- qvm_impl<T, group_size, bits>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- K,
- N,
- tid,
- simd_gid,
- simd_lid
- );
-}
-
-template <typename T, int group_size, int bits>
-[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_fast(
- const device uint32_t* w [[buffer(0)]],
- const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device T* x [[buffer(3)]],
- device T* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- qmv_fast_impl<T, group_size, bits>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- K,
- N,
- tid,
- simd_gid,
- simd_lid
- );
-}
-
-template [[host_name("qmv_f16_g32_b4_fast")]] [[kernel]] void qmv_fast<
- half,
- 32,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g64_b4_fast")]] [[kernel]] void qmv_fast<
- half,
- 64,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g128_b4_fast")]] [[kernel]] void qmv_fast<
- half,
- 128,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g32_b4_fast")]] [[kernel]] void qmv_fast<
- bfloat,
- 32,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g64_b4_fast")]] [[kernel]] void qmv_fast<
- bfloat,
- 64,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g128_b4_fast")]] [[kernel]] void qmv_fast<
- bfloat,
- 128,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g32_b4_fast")]] [[kernel]] void qmv_fast<
- float,
- 32,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g64_b4_fast")]] [[kernel]] void qmv_fast<
- float,
- 64,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g128_b4_fast")]] [[kernel]] void qmv_fast<
- float,
- 128,
- 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g32_b8_fast")]] [[kernel]] void qmv_fast<
- half,
- 32,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g64_b8_fast")]] [[kernel]] void qmv_fast<
- half,
- 64,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g128_b8_fast")]] [[kernel]] void qmv_fast<
- half,
- 128,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g32_b8_fast")]] [[kernel]] void qmv_fast<
- bfloat,
- 32,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g64_b8_fast")]] [[kernel]] void qmv_fast<
- bfloat,
- 64,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g128_b8_fast")]] [[kernel]] void qmv_fast<
- bfloat,
- 128,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g32_b8_fast")]] [[kernel]] void qmv_fast<
- float,
- 32,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g64_b8_fast")]] [[kernel]] void qmv_fast<
- float,
- 64,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g128_b8_fast")]] [[kernel]] void qmv_fast<
- float,
- 128,
- 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 32 (F16)
-template [[host_name("qmm_f16_g32_b4")]] [[kernel]] void qmm<
- half,
- 32,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g32_b4")]] [[kernel]] void
-qmm_transposed<half, 32, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]] [[kernel]] void
-qmm_transposed<half, 32, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f16_g32_b4_alignedk")]] [[kernel]] void qmm<
- half,
- 32,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g32_b4")]] [[kernel]] void qmv<half, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f16_g32_b4")]] [[kernel]] void qvm<half, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 32 (F16) - 8 bit
-template [[host_name("qmm_f16_g32_b8")]] [[kernel]] void qmm<
- half,
- 32,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g32_b8")]] [[kernel]] void
-qmm_transposed<half, 32, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]] [[kernel]] void
-qmm_transposed<half, 32, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f16_g32_b8_alignedk")]] [[kernel]] void qmm<
- half,
- 32,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g32_b8")]] [[kernel]] void qmv<half, 32, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f16_g32_b8")]] [[kernel]] void qvm<half, 32, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 64 (F16)
-template [[host_name("qmm_f16_g64_b4")]] [[kernel]] void qmm<
- half,
- 64,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f16_g64_b4_alignedk")]] [[kernel]] void qmm<
- half,
- 64,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g64_b4")]] [[kernel]] void
-qmm_transposed<half, 64, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]] [[kernel]] void
-qmm_transposed<half, 64, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g64_b4")]] [[kernel]] void qmv<half, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f16_g64_b4")]] [[kernel]] void qvm<half, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 64 (F16) - 8 bit
-template [[host_name("qmm_f16_g64_b8")]] [[kernel]] void qmm<
- half,
- 64,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f16_g64_b8_alignedk")]] [[kernel]] void qmm<
- half,
- 64,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g64_b8")]] [[kernel]] void
-qmm_transposed<half, 64, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]] [[kernel]] void
-qmm_transposed<half, 64, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g64_b8")]] [[kernel]] void qmv<half, 64, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f16_g64_b8")]] [[kernel]] void qvm<half, 64, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 128 (F16)
-template [[host_name("qmm_f16_g128_b4")]] [[kernel]] void qmm<
- half,
- 128,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f16_g128_b4_alignedk")]] [[kernel]] void qmm<
- half,
- 128,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g128_b4")]] [[kernel]] void
-qmm_transposed<half, 128, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]] [[kernel]] void
-qmm_transposed<half, 128, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g128_b4")]] [[kernel]] void qmv<half, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f16_g128_b4")]] [[kernel]] void qvm<half, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 128 (F16) - 8 bit
-template [[host_name("qmm_f16_g128_b8")]] [[kernel]] void qmm<
- half,
- 128,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f16_g128_b8_alignedk")]] [[kernel]] void qmm<
- half,
- 128,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g128_b8")]] [[kernel]] void
-qmm_transposed<half, 128, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]] [[kernel]] void
-qmm_transposed<half, 128, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f16_g128_b8")]] [[kernel]] void qmv<half, 128, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f16_g128_b8")]] [[kernel]] void qvm<half, 128, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 32 (F32)
-template [[host_name("qmm_f32_g32_b4")]] [[kernel]] void qmm<
- float,
- 32,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g32_b4")]] [[kernel]] void
-qmm_transposed<float, 32, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]] [[kernel]] void
-qmm_transposed<float, 32, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f32_g32_b4_alignedk")]] [[kernel]] void qmm<
- float,
- 32,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g32_b4")]] [[kernel]] void qmv<float, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f32_g32_b4")]] [[kernel]] void qvm<float, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 32 (F32) - 8 bit
-template [[host_name("qmm_f32_g32_b8")]] [[kernel]] void qmm<
- float,
- 32,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g32_b8")]] [[kernel]] void
-qmm_transposed<float, 32, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]] [[kernel]] void
-qmm_transposed<float, 32, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f32_g32_b8_alignedk")]] [[kernel]] void qmm<
- float,
- 32,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g32_b8")]] [[kernel]] void qmv<float, 32, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f32_g32_b8")]] [[kernel]] void qvm<float, 32, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 64 (F32)
-template [[host_name("qmm_f32_g64_b4")]] [[kernel]] void qmm<
- float,
- 64,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f32_g64_b4_alignedk")]] [[kernel]] void qmm<
- float,
- 64,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g64_b4")]] [[kernel]] void
-qmm_transposed<float, 64, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]] [[kernel]] void
-qmm_transposed<float, 64, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g64_b4")]] [[kernel]] void qmv<float, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f32_g64_b4")]] [[kernel]] void qvm<float, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 64 (F32) - 8 bit
-template [[host_name("qmm_f32_g64_b8")]] [[kernel]] void qmm<
- float,
- 64,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f32_g64_b8_alignedk")]] [[kernel]] void qmm<
- float,
- 64,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g64_b8")]] [[kernel]] void
-qmm_transposed<float, 64, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]] [[kernel]] void
-qmm_transposed<float, 64, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g64_b8")]] [[kernel]] void qmv<float, 64, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f32_g64_b8")]] [[kernel]] void qvm<float, 64, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 128 (F32)
-template [[host_name("qmm_f32_g128_b4")]] [[kernel]] void qmm<
- float,
- 128,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f32_g128_b4_alignedk")]] [[kernel]] void qmm<
- float,
- 128,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g128_b4")]] [[kernel]] void
-qmm_transposed<float, 128, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]] [[kernel]] void
-qmm_transposed<float, 128, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g128_b4")]] [[kernel]] void qmv<float, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f32_g128_b4")]] [[kernel]] void qvm<float, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 128 (F32) - 8 bit
-template [[host_name("qmm_f32_g128_b8")]] [[kernel]] void qmm<
- float,
- 128,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_f32_g128_b8_alignedk")]] [[kernel]] void qmm<
- float,
- 128,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g128_b8")]] [[kernel]] void
-qmm_transposed<float, 128, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]] [[kernel]] void
-qmm_transposed<float, 128, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_f32_g128_b8")]] [[kernel]] void qmv<float, 128, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device float* biases [[buffer(2)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_f32_g128_b8")]] [[kernel]] void qvm<float, 128, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device float* x [[buffer(3)]],
- device float* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 32 (BF16)
-template [[host_name("qmm_bf16_g32_b4")]] [[kernel]] void qmm<
- bfloat,
- 32,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_bf16_g32_b4_alignedk")]] [[kernel]] void qmm<
- bfloat,
- 32,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g32_b4")]] [[kernel]] void
-qmm_transposed<bfloat, 32, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]] [[kernel]] void
-qmm_transposed<bfloat, 32, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g32_b4")]] [[kernel]] void qmv<bfloat, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_bf16_g32_b4")]] [[kernel]] void qvm<bfloat, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 32 (BF16) - 8 bit
-template [[host_name("qmm_bf16_g32_b8")]] [[kernel]] void qmm<
- bfloat,
- 32,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_bf16_g32_b8_alignedk")]] [[kernel]] void qmm<
- bfloat,
- 32,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g32_b8")]] [[kernel]] void
-qmm_transposed<bfloat, 32, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]] [[kernel]] void
-qmm_transposed<bfloat, 32, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g32_b8")]] [[kernel]] void qmv<bfloat, 32, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_bf16_g32_b8")]] [[kernel]] void qvm<bfloat, 32, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 64 (BF16)
-template [[host_name("qmm_bf16_g64_b4")]] [[kernel]] void qmm<
- bfloat,
- 64,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_bf16_g64_b4_alignedk")]] [[kernel]] void qmm<
- bfloat,
- 64,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g64_b4")]] [[kernel]] void
-qmm_transposed<bfloat, 64, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]] [[kernel]] void
-qmm_transposed<bfloat, 64, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g64_b4_64x64")]] [[kernel]] void
-qmm_transposed<bfloat, 64, 4, false, 64, 32, 64>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g64_b4")]] [[kernel]] void qmv<bfloat, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_bf16_g64_b4")]] [[kernel]] void qvm<bfloat, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 64 (BF16) - 8 bit
-template [[host_name("qmm_bf16_g64_b8")]] [[kernel]] void qmm<
- bfloat,
- 64,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_bf16_g64_b8_alignedk")]] [[kernel]] void qmm<
- bfloat,
- 64,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g64_b8")]] [[kernel]] void
-qmm_transposed<bfloat, 64, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]] [[kernel]] void
-qmm_transposed<bfloat, 64, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g64_b8_64x64")]] [[kernel]] void
-qmm_transposed<bfloat, 64, 8, false, 64, 32, 64>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g64_b8")]] [[kernel]] void qmv<bfloat, 64, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_bf16_g64_b8")]] [[kernel]] void qvm<bfloat, 64, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 128 (BF16)
-template [[host_name("qmm_bf16_g128_b4")]] [[kernel]] void qmm<
- bfloat,
- 128,
- 4,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_bf16_g128_b4_alignedk")]] [[kernel]] void qmm<
- bfloat,
- 128,
- 4,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b4")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 4, true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 4, false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g128_b4")]] [[kernel]] void qmv<bfloat, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_bf16_g128_b4")]] [[kernel]] void qvm<bfloat, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-// Group size 128 (BF16) - 8 bit
-template [[host_name("qmm_bf16_g128_b8")]] [[kernel]] void qmm<
- bfloat,
- 128,
- 8,
- false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_bf16_g128_b8_alignedk")]] [[kernel]] void qmm<
- bfloat,
- 128,
- 8,
- true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b8")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 8, true>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 8, false>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_bf16_g128_b8")]] [[kernel]] void qmv<bfloat, 128, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qvm_bf16_g128_b8")]] [[kernel]] void qvm<bfloat, 128, 8>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b8_64x64")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 8, true, 64, 32, 64>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]] [[kernel]] void
-qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& N [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-///////////////////////////////////////////////////////////////////////////////
-/// MLP Fused QMV - Quantized Matrix-Vector with Fused Activation
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename U>
-static inline U extract_zp(
- const device uint8_t* zps,
- int row_offset,
- int zp_stride,
- int bits,
- bool high_nibble
-) {
- uint8_t zp_byte = zps[row_offset * zp_stride];
- return bits == 4
- ? static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F))
- : static_cast<U>(zp_byte);
-};
-
-// MLP fused QMV kernel for decode path (M=1).
-// Computes paired up and gate projections, then applies: out = up *
-// activation(gate) Weight layout: [up_weights (hidden_dim rows), gate_weights
-// (hidden_dim rows)] Output size is hidden_dim.
-template <typename T, int group_size, int bits>
-void qmv_mlp_fused_impl(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* x,
- device T* y,
- const constant int& in_vec_size,
- const constant int& hidden_dim, // Output size (half of weight rows)
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- constexpr int packs_per_thread = bits == 2 ? 1 : 2;
- constexpr int num_simdgroups = 2;
- constexpr int results_per_simdgroup = 4; // Compute 4 paired rows at a time
- constexpr int pack_factor = get_pack_factor<bits, 32>();
- constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
- constexpr int values_per_thread = pack_factor * packs_per_thread;
- constexpr int block_size = values_per_thread * SIMD_SIZE;
- constexpr int scale_step_per_thread = group_size / values_per_thread;
- const device uint8_t* ws = (const device uint8_t*)w;
- typedef float U;
- thread U x_thread[values_per_thread];
- thread U result_up[results_per_simdgroup] = {0};
- thread U result_gate[results_per_simdgroup] = {0};
-
- const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
- const int in_vec_size_g = in_vec_size / group_size;
-
- // Output row for 'up' projection
- const int out_row_up = tid.y * (num_simdgroups * results_per_simdgroup) +
- simd_gid * results_per_simdgroup;
-
- // Corresponding 'gate' row is offset by hidden_dim
- const int out_row_gate = out_row_up + hidden_dim;
-
- // Check bounds - only process if up row is within hidden_dim
- if (out_row_up >= hidden_dim)
- return;
-
- // Pointer setup for up weights
- const device uint8_t* ws_up = ws + out_row_up * in_vec_size_w +
- simd_lid * packs_per_thread * bytes_per_pack;
- const device T* scales_up =
- scales + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;
-
- // Pointer setup for gate weights
- const device uint8_t* ws_gate = ws + out_row_gate * in_vec_size_w +
- simd_lid * packs_per_thread * bytes_per_pack;
- const device T* scales_gate =
- scales + out_row_gate * in_vec_size_g + simd_lid / scale_step_per_thread;
-
- const device T* biases_up = nullptr;
- const device T* biases_gate = nullptr;
- const device uint8_t* zps_up = nullptr;
- const device uint8_t* zps_gate = nullptr;
- int zp_stride = 0;
- bool high_nibble = false;
-
- if (kUseMlxQuant) {
- biases_up =
- biases + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;
- biases_gate = biases + out_row_gate * in_vec_size_g +
- simd_lid / scale_step_per_thread;
- } else {
- if (bits == 4) {
- zp_stride = (in_vec_size_g + 1) / 2;
- zps_up = zero_points + out_row_up * zp_stride;
- zps_gate = zero_points + out_row_gate * zp_stride;
- int g_offset = simd_lid / scale_step_per_thread;
- zps_up += g_offset / 2;
- zps_gate += g_offset / 2;
- high_nibble = (g_offset & 1);
- } else {
- zp_stride = in_vec_size_g;
- zps_up = zero_points + out_row_up * zp_stride +
- simd_lid / scale_step_per_thread;
- zps_gate = zero_points + out_row_gate * zp_stride +
- simd_lid / scale_step_per_thread;
- }
- }
-
- x += tid.x * in_vec_size + simd_lid * values_per_thread;
- y += tid.x * hidden_dim + out_row_up;
-
- for (int k = 0; k < in_vec_size; k += block_size) {
- U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
-
- // Compute both up and gate dot products for 4 rows each
- {
- auto wl_up0 = ws_up;
- auto wl_up1 = ws_up + in_vec_size_w;
- auto wl_up2 = ws_up + 2 * in_vec_size_w;
- auto wl_up3 = ws_up + 3 * in_vec_size_w;
-
- auto wl_gate0 = ws_gate;
- auto wl_gate1 = ws_gate + in_vec_size_w;
- auto wl_gate2 = ws_gate + 2 * in_vec_size_w;
- auto wl_gate3 = ws_gate + 3 * in_vec_size_w;
-
- U s_up0 = static_cast<U>(scales_up[0]);
- U s_up1 = static_cast<U>(scales_up[in_vec_size_g]);
- U s_up2 = static_cast<U>(scales_up[2 * in_vec_size_g]);
- U s_up3 = static_cast<U>(scales_up[3 * in_vec_size_g]);
-
- U s_gate0 = static_cast<U>(scales_gate[0]);
- U s_gate1 = static_cast<U>(scales_gate[in_vec_size_g]);
- U s_gate2 = static_cast<U>(scales_gate[2 * in_vec_size_g]);
- U s_gate3 = static_cast<U>(scales_gate[3 * in_vec_size_g]);
-
- if (kUseMlxQuant) {
- U b_up0 = static_cast<U>(biases_up[0]);
- U b_up1 = static_cast<U>(biases_up[in_vec_size_g]);
- U b_up2 = static_cast<U>(biases_up[2 * in_vec_size_g]);
- U b_up3 = static_cast<U>(biases_up[3 * in_vec_size_g]);
-
- U b_gate0 = static_cast<U>(biases_gate[0]);
- U b_gate1 = static_cast<U>(biases_gate[in_vec_size_g]);
- U b_gate2 = static_cast<U>(biases_gate[2 * in_vec_size_g]);
- U b_gate3 = static_cast<U>(biases_gate[3 * in_vec_size_g]);
-
- result_up[0] += qdot<U, values_per_thread, bits>(
- wl_up0,
- x_thread,
- s_up0,
- b_up0,
- sum
- );
- result_up[1] += qdot<U, values_per_thread, bits>(
- wl_up1,
- x_thread,
- s_up1,
- b_up1,
- sum
- );
- result_up[2] += qdot<U, values_per_thread, bits>(
- wl_up2,
- x_thread,
- s_up2,
- b_up2,
- sum
- );
- result_up[3] += qdot<U, values_per_thread, bits>(
- wl_up3,
- x_thread,
- s_up3,
- b_up3,
- sum
- );
-
- result_gate[0] += qdot<U, values_per_thread, bits>(
- wl_gate0,
- x_thread,
- s_gate0,
- b_gate0,
- sum
- );
- result_gate[1] += qdot<U, values_per_thread, bits>(
- wl_gate1,
- x_thread,
- s_gate1,
- b_gate1,
- sum
- );
- result_gate[2] += qdot<U, values_per_thread, bits>(
- wl_gate2,
- x_thread,
- s_gate2,
- b_gate2,
- sum
- );
- result_gate[3] += qdot<U, values_per_thread, bits>(
- wl_gate3,
- x_thread,
- s_gate3,
- b_gate3,
- sum
- );
- } else {
- U zp_up0 = extract_zp<U>(zps_up, 0, zp_stride, bits, high_nibble);
- U zp_up1 = extract_zp<U>(zps_up, 1, zp_stride, bits, high_nibble);
- U zp_up2 = extract_zp<U>(zps_up, 2, zp_stride, bits, high_nibble);
- U zp_up3 = extract_zp<U>(zps_up, 3, zp_stride, bits, high_nibble);
-
- U zp_gate0 = extract_zp<U>(zps_gate, 0, zp_stride, bits, high_nibble);
- U zp_gate1 = extract_zp<U>(zps_gate, 1, zp_stride, bits, high_nibble);
- U zp_gate2 = extract_zp<U>(zps_gate, 2, zp_stride, bits, high_nibble);
- U zp_gate3 = extract_zp<U>(zps_gate, 3, zp_stride, bits, high_nibble);
-
- result_up[0] += qdot_zero_point<U, values_per_thread, bits>(
- wl_up0,
- x_thread,
- s_up0,
- zp_up0
- );
- result_up[1] += qdot_zero_point<U, values_per_thread, bits>(
- wl_up1,
- x_thread,
- s_up1,
- zp_up1
- );
- result_up[2] += qdot_zero_point<U, values_per_thread, bits>(
- wl_up2,
- x_thread,
- s_up2,
- zp_up2
- );
- result_up[3] += qdot_zero_point<U, values_per_thread, bits>(
- wl_up3,
- x_thread,
- s_up3,
- zp_up3
- );
-
- result_gate[0] += qdot_zero_point<U, values_per_thread, bits>(
- wl_gate0,
- x_thread,
- s_gate0,
- zp_gate0
- );
- result_gate[1] += qdot_zero_point<U, values_per_thread, bits>(
- wl_gate1,
- x_thread,
- s_gate1,
- zp_gate1
- );
- result_gate[2] += qdot_zero_point<U, values_per_thread, bits>(
- wl_gate2,
- x_thread,
- s_gate2,
- zp_gate2
- );
- result_gate[3] += qdot_zero_point<U, values_per_thread, bits>(
- wl_gate3,
- x_thread,
- s_gate3,
- zp_gate3
- );
- }
- }
-
- ws_up += block_size * bytes_per_pack / pack_factor;
- ws_gate += block_size * bytes_per_pack / pack_factor;
- scales_up += block_size / group_size;
- scales_gate += block_size / group_size;
- if (kUseMlxQuant) {
- biases_up += block_size / group_size;
- biases_gate += block_size / group_size;
- } else {
- if (bits == 4) {
- zps_up += (block_size / group_size) / 2;
- zps_gate += (block_size / group_size) / 2;
- } else {
- zps_up += block_size / group_size;
- zps_gate += block_size / group_size;
- }
- }
- x += block_size;
- }
-
- // SIMD reduction and fused activation
- for (int row = 0; row < results_per_simdgroup; row++) {
- result_up[row] = simd_sum(result_up[row]);
- result_gate[row] = simd_sum(result_gate[row]);
- if (simd_lid == 0) {
- // Apply MLP fused epilogue: up * activation(gate)
- float fused = mlp_fused_epilogue_f32(result_up[row], result_gate[row]);
- y[row] = static_cast<T>(fused);
- }
- }
-}
-
-template <typename T, int group_size, int bits>
-[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_mlp_fused(
- const device uint32_t* w [[buffer(0)]],
- const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device T* x [[buffer(3)]],
- device T* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- qmv_mlp_fused_impl<T, group_size, bits>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- K,
- hidden_dim,
- tid,
- simd_gid,
- simd_lid
- );
-}
-
-// Instantiate MLP fused QMV kernels for common configurations
-template [[host_name("qmv_mlp_fused_f16_g32_b4")]] [[kernel]] void
-qmv_mlp_fused<half, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_mlp_fused_f16_g64_b4")]] [[kernel]] void
-qmv_mlp_fused<half, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_mlp_fused_f16_g128_b4")]] [[kernel]] void
-qmv_mlp_fused<half, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_mlp_fused_bf16_g32_b4")]] [[kernel]] void
-qmv_mlp_fused<bfloat, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_mlp_fused_bf16_g64_b4")]] [[kernel]] void
-qmv_mlp_fused<bfloat, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmv_mlp_fused_bf16_g128_b4")]] [[kernel]] void
-qmv_mlp_fused<bfloat, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-///////////////////////////////////////////////////////////////////////////////
-/// MLP Fused QMM - Quantized Matrix-Matrix with Fused Activation
-///////////////////////////////////////////////////////////////////////////////
-
-// MLP fused QMM kernel for prefill path (M>1).
-// Computes paired up and gate projections, then applies: out = up *
-// activation(gate) Weight layout: [up_weights (hidden_dim cols), gate_weights
-// (hidden_dim cols)] Output size is [M, hidden_dim].
-template <
- typename T,
- int group_size,
- int bits,
- bool aligned_K,
- int BM = 32,
- int BK = 32,
- int BN = 32>
-void qmm_mlp_fused_impl(
- const device uint32_t* w,
- const device T* scales,
- const device uint8_t* zero_points,
- const device T* biases,
- const device T* x,
- device T* y,
- threadgroup T* Xs,
- threadgroup T* Ws_up,
- const constant int& K,
- const constant int&
- hidden_dim, // Output columns (half of total weight columns)
- const constant int& M,
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
- static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
-
- (void)lid;
-
- constexpr int WM = 2;
- constexpr int WN = 2;
- constexpr int pack_factor = get_pack_factor<bits, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
-
- constexpr int BK_padded = (BK + 16 / sizeof(T));
- constexpr int BN_padded = (BN + 16 / sizeof(T));
-
- using mma_t = matmul_utils::
- BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;
- using loader_x_t =
- matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;
-
- auto wl_up = (const device uint8_t*)w;
-
- const int y_row = tid.y * BM;
- const int y_col = tid.x * BN; // Column in output (within hidden_dim)
-
- // Only process tiles within hidden_dim
- if (y_col >= hidden_dim)
- return;
-
- x += y_row * static_cast<int64_t>(K);
-
- // Up weights at column y_col
- wl_up += y_col * bytes_per_pack / pack_factor;
- // Gate weights at column y_col + hidden_dim
- auto wl_gate = wl_up + hidden_dim * bytes_per_pack / pack_factor;
-
- y += y_row * static_cast<int64_t>(hidden_dim) + y_col;
-
- const device T* scales_base = scales;
- const device T* biases_base = biases;
-
- const short num_els = min(BM, M - y_row);
- const short num_outs = min(BN, hidden_dim - y_col);
-
- // Threadgroup memory for gate weights (after Ws_up)
- threadgroup T* Ws_gate = Ws_up + BK * BN_padded;
-
- loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);
- mma_t mma_up(simd_gid, simd_lid);
- mma_t mma_gate(simd_gid, simd_lid);
-
- const int k_iterations = (K + BK - 1) / BK;
- const int full_iterations = aligned_K ? k_iterations : k_iterations - 1;
-
- // Total output width for scale/bias strides
- const int N_total = hidden_dim * 2;
-
- if (kUseMlxQuant) {
- using loader_w_t = QuantizedBlockLoaderMlx<
- T,
- BK,
- BN,
- BN_padded,
- 0,
- WM * WN * 32,
- group_size,
- bits>;
-
- const device T* scales_up = scales_base + (y_col / group_size);
- const device T* biases_up = biases_base + (y_col / group_size);
- const device T* scales_gate =
- scales_base + ((y_col + hidden_dim) / group_size);
- const device T* biases_gate =
- biases_base + ((y_col + hidden_dim) / group_size);
-
- loader_w_t loader_w_up(
- wl_up,
- scales_up,
- biases_up,
- N_total,
- Ws_up,
- simd_gid,
- simd_lid
- );
- loader_w_t loader_w_gate(
- wl_gate,
- scales_gate,
- biases_gate,
- N_total,
- Ws_gate,
- simd_gid,
- simd_lid
- );
-
- for (int k = 0; k < full_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_unsafe();
- loader_w_up.load_unsafe();
- loader_w_gate.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_up.mma(Xs, Ws_up);
- mma_gate.mma(Xs, Ws_gate);
-
- loader_x.next();
- loader_w_up.next();
- loader_w_gate.next();
- }
-
- if (!aligned_K) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(BM, K - full_iterations * BK));
- loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));
- loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_up.mma(Xs, Ws_up);
- mma_gate.mma(Xs, Ws_gate);
- }
- } else {
- const int out_groups_total = (N_total + group_size - 1) / group_size;
- const int groups_per_row = out_groups_total;
- const int out_group_up = y_col / group_size;
- const int out_group_gate = (y_col + hidden_dim) / group_size;
- const int zp_stride_out =
- (bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;
-
- using loader_w_t = QuantizedBlockLoaderZp<
- T,
- BK,
- BN,
- BN_padded,
- 0,
- WM * WN * 32,
- group_size,
- bits,
- true>;
-
- loader_w_t loader_w_up(
- wl_up,
- scales_base,
- zero_points,
- N_total,
- groups_per_row,
- Ws_up,
- simd_gid,
- simd_lid,
- out_group_up,
- out_groups_total,
- zp_stride_out
- );
-
- loader_w_t loader_w_gate(
- wl_gate,
- scales_base,
- zero_points,
- N_total,
- groups_per_row,
- Ws_gate,
- simd_gid,
- simd_lid,
- out_group_gate,
- out_groups_total,
- zp_stride_out
- );
-
- for (int k = 0; k < full_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_unsafe();
- loader_w_up.load_unsafe();
- loader_w_gate.load_unsafe();
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_up.mma(Xs, Ws_up);
- mma_gate.mma(Xs, Ws_gate);
-
- loader_x.next();
- loader_w_up.next();
- loader_w_gate.next();
- }
-
- if (!aligned_K) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_x.load_safe(short2(BM, K - full_iterations * BK));
- loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));
- loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_up.mma(Xs, Ws_up);
- mma_gate.mma(Xs, Ws_gate);
- }
- }
-
- // Apply MLP fused epilogue: out = up * activation(gate)
- // Access Ctile elements and apply fusion
- constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;
-#pragma unroll
- for (short i = 0; i < kElemsPerTile; i++) {
- float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);
- float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);
- float fused = mlp_fused_epilogue_f32(up_val, gate_val);
- mma_up.Ctile.elems()[i] = static_cast<T>(fused);
- }
-
- // Store result
- mma_up.store_result_safe(y, hidden_dim, short2(num_outs, num_els));
-}
-
-template <typename T, int group_size, int bits>
-[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_mlp_fused(
- const device uint32_t* w [[buffer(0)]],
- const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points
- [[buffer(2), function_constant(kUseZeroPoints)]],
- const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
- const device T* x [[buffer(3)]],
- device T* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- constexpr int BM = 32;
- constexpr int BK = 32;
- constexpr int BN = 32;
- constexpr int BK_padded = (BK + 16 / sizeof(T));
- constexpr int BN_padded = (BN + 16 / sizeof(T));
-
- threadgroup T Xs[BM * BK_padded];
- threadgroup T Ws[2 * BK * BN_padded]; // Double for up and gate
-
- bool aligned_K = (K % BK) == 0;
- if (aligned_K) {
- qmm_mlp_fused_impl<T, group_size, bits, true, BM, BK, BN>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- K,
- hidden_dim,
- M,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
- } else {
- qmm_mlp_fused_impl<T, group_size, bits, false, BM, BK, BN>(
- w,
- scales,
- zero_points,
- biases,
- x,
- y,
- Xs,
- Ws,
- K,
- hidden_dim,
- M,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
- }
-}
-
-// Instantiate MLP fused QMM kernels
-template [[host_name("qmm_mlp_fused_f16_g32_b4")]] [[kernel]] void
-qmm_mlp_fused<half, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_mlp_fused_f16_g64_b4")]] [[kernel]] void
-qmm_mlp_fused<half, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_mlp_fused_f16_g128_b4")]] [[kernel]] void
-qmm_mlp_fused<half, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device half* biases [[buffer(2)]],
- const device half* x [[buffer(3)]],
- device half* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_mlp_fused_bf16_g32_b4")]] [[kernel]] void
-qmm_mlp_fused<bfloat, 32, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_mlp_fused_bf16_g64_b4")]] [[kernel]] void
-qmm_mlp_fused<bfloat, 64, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
-
-template [[host_name("qmm_mlp_fused_bf16_g128_b4")]] [[kernel]] void
-qmm_mlp_fused<bfloat, 128, 4>(
- const device uint32_t* w [[buffer(0)]],
- const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2)]],
- const device bfloat* biases [[buffer(2)]],
- const device bfloat* x [[buffer(3)]],
- device bfloat* y [[buffer(4)]],
- const constant int& K [[buffer(5)]],
- const constant int& hidden_dim [[buffer(6)]],
- const constant int& M [[buffer(7)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint lid [[thread_index_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-);
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
new file mode 100644
index 000000000..e69a9123c
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
@@ -0,0 +1,60 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+#include "common.h"
+
+template <typename T, int group_size, int bits>
+VARIANTS(T, float, half, bfloat)
+VARIANTS(group_size, 32, 64, 128)
+VARIANTS(bits, 4, 8)
+KERNEL(QuantizedMatmulQvm)(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points OPTIONAL(use_zero_points),
+ const device T* biases OPTIONAL(use_mlx_quant),
+ const device T* x,
+ device T* y,
+ const constant int& k,
+ const constant int& n,
+ const constant int& m,
+ const bool use_zero_points SPECIALIZE,
+ const bool use_mlx_quant SPECIALIZE,
+ const uint tgid_x GROUPS(m),
+ const uint tgid_y GROUPS((n + 64 - 1) / 64),
+ const uint tgid_z GROUPS(1),
+ const uint tid_x THREADS(32),
+ const uint tid_y THREADS(2)
+) {
+ const uint3 tid = uint3(tgid_x, tgid_y, tgid_z);
+ const uint simd_gid = tid_y;
+ const uint simd_lid = tid_x;
+
+ if (use_mlx_quant) {
+ qvm_impl<T, group_size, bits, true>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ k,
+ n,
+ tid,
+ simd_gid,
+ simd_lid
+ );
+ } else {
+ qvm_impl<T, group_size, bits, false>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ k,
+ n,
+ tid,
+ simd_gid,
+ simd_lid
+ );
+ }
+}
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index fd3fa8e24..7f07f6282 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -7,11 +7,11 @@ use objc2::{rc::Retained, runtime::ProtocolObject};
use uzu::{
DataType,
backends::{
- common::Context,
- metal::{
- MetalContext,
- kernel::quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},
+ common::{
+ Context,
+ kernel::matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulType},
},
+ metal::{MetalContext, kernel::quant_matmul::QuantizedMatmulKernel},
},
config::QuantizationMode,
};
@@ -152,7 +152,7 @@ fn cpu_reference(
group_size: usize,
dtype: DataType,
bits: usize,
- quantization_type: QuantizationType,
+ quantization_type: QuantizedMatmulType,
zero_points: &[u8],
zero_points_stride: usize,
) -> Vec<f32> {
@@ -185,7 +185,7 @@ fn cpu_reference(
let val_a = a[i * input_dim + l];
let scale = scales[l * num_groups + group_idx];
- let bias = if quantization_type == QuantizationType::ZeroPoint {
+ let bias = if quantization_type == QuantizedMatmulType::ZeroPoint {
let zp_val_qvm = get_zp_value(zero_points, zero_points_stride, l, group_idx, bits);
-scale * zp_val_qvm
} else {
@@ -197,11 +197,11 @@ fn cpu_reference(
for g in 0..num_groups {
let scale = scales[j * num_groups + g];
let bias = match quantization_type {
- QuantizationType::ZeroPoint => {
+ QuantizedMatmulType::ZeroPoint => {
let zp = get_zp_value(zero_points, zero_points_stride, j, g, bits);
-scale * zp
},
- QuantizationType::Mlx => biases[j * num_groups + g],
+ QuantizedMatmulType::Mlx => biases[j * num_groups + g],
};
let l_start = g * group_size;
let l_end = (l_start + group_size).min(input_dim);
@@ -247,7 +247,7 @@ fn generate_test_quant_params(
group_size: usize,
data_type: DataType,
bits: usize,
- quantization_type: QuantizationType,
+ quantization_type: QuantizedMatmulType,
weights_transposed: bool,
randomize_zp: bool,
) -> TestQuantParams {
@@ -281,7 +281,7 @@ fn generate_test_quant_params(
};
let mut zero_points = vec![0u8; zero_points_len];
- if quantization_type == QuantizationType::ZeroPoint && randomize_zp {
+ if quantization_type == QuantizedMatmulType::ZeroPoint && randomize_zp {
if !weights_transposed {
// GMM/QVM non-transposed: Use [K][N_groups] layout for generation
for k in 0..input_dim {
@@ -352,7 +352,7 @@ fn generate_test_quant_params(
}
}
- if quantization_type == QuantizationType::Mlx {
+ if quantization_type == QuantizedMatmulType::Mlx {
if !weights_transposed {
for k in 0..input_dim {
for g in 0..num_groups {
@@ -421,7 +421,7 @@ fn execute_quantized_matmul(
weights_transposed: bool,
iterations: usize,
validate: bool,
- quantization_type: QuantizationType,
+ quantization_type: QuantizedMatmulType,
randomize_zp: bool,
group_size: usize,
data_type: DataType,
@@ -466,28 +466,30 @@ fn execute_quantized_matmul(
let s_buf = buffer_from_f32_slice(ctx, data_type, &params.scales);
let b_buf = match quantization_type {
- QuantizationType::ZeroPoint => ctx
+ QuantizedMatmulType::ZeroPoint => ctx
.device
.new_buffer_with_data(&params.zero_points, MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer"),
- QuantizationType::Mlx => buffer_from_f32_slice(ctx, data_type, &params.biases),
+ QuantizedMatmulType::Mlx => buffer_from_f32_slice(ctx, data_type, &params.biases),
};
let x_buf = buffer_from_f32_slice(ctx, data_type, &x_f32);
let y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");
let kernel = QuantizedMatmulKernel::new(
&ctx,
- data_type,
- group_size,
- input_dim,
- output_dim,
- match bits {
- 4 => QuantizationMode::UInt4,
- 8 => QuantizationMode::Int8,
- _ => panic!("Unsupported bits: {}", bits),
+ QuantizedMatmulConfiguration {
+ data_type,
+ group_size,
+ input_dim,
+ output_dim,
+ mode: match bits {
+ 4 => QuantizationMode::UInt4,
+ 8 => QuantizationMode::Int8,
+ _ => panic!("Unsupported bits: {}", bits),
+ },
+ quantization_type,
+ weights_transposed,
},
- quantization_type,
- weights_transposed,
)
.unwrap();
@@ -500,9 +502,9 @@ fn execute_quantized_matmul(
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
output_buffer: &y_buf,
- batch: batch as i32,
- input_dim: input_dim as i32,
- output_dim: output_dim as i32,
+ batch,
+ input_dim,
+ output_dim,
quantization_type,
};
let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
@@ -523,9 +525,9 @@ fn execute_quantized_matmul(
scales_buffer: &s_buf,
zero_points_or_biases_buffer: &b_buf,
output_buffer: &y_buf,
- batch: batch as i32,
- input_dim: input_dim as i32,
- output_dim: output_dim as i32,
+ batch,
+ input_dim,
+ output_dim,
quantization_type,
};
let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
@@ -620,7 +622,7 @@ fn execute_quantized_matmul(
}
struct TestConfig {
- quant_type: QuantizationType,
+ quant_type: QuantizedMatmulType,
bits: usize,
data_type: DataType,
group_size: usize,
@@ -640,7 +642,7 @@ fn run_kernel_test(
validate: bool,
iterations: usize,
) -> ExecutionResult {
- let randomize_zp = config.quant_type == QuantizationType::ZeroPoint;
+ let randomize_zp = config.quant_type == QuantizedMatmulType::ZeroPoint;
execute_quantized_matmul(
ctx,
@@ -670,25 +672,25 @@ fn test_quant_gmv() {
let configs = vec![
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 8,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 8,
data_type: DataType::F32,
group_size: 64,
@@ -714,25 +716,25 @@ fn test_quant_qvm() {
let configs = vec![
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 8,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 8,
data_type: DataType::F32,
group_size: 64,
@@ -758,25 +760,25 @@ fn test_quant_gmm() {
let configs = vec![
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 8,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 8,
data_type: DataType::F32,
group_size: 64,
@@ -802,25 +804,25 @@ fn test_quant_gmm_transposed() {
let configs = vec![
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 8,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 4,
data_type: DataType::F32,
group_size: 64,
},
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 8,
data_type: DataType::F32,
group_size: 64,
@@ -858,28 +860,28 @@ fn test_quant_matmul_perf() {
let configs = vec![
// 4-bit Mlx BF16
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 4,
data_type: DataType::BF16,
group_size: 128,
},
// 8-bit Mlx BF16
TestConfig {
- quant_type: QuantizationType::Mlx,
+ quant_type: QuantizedMatmulType::Mlx,
bits: 8,
data_type: DataType::BF16,
group_size: 128,
},
// 4-bit ZP F16
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 4,
data_type: DataType::F16,
group_size: 64,
},
// 8-bit ZP F16
TestConfig {
- quant_type: QuantizationType::ZeroPoint,
+ quant_type: QuantizedMatmulType::ZeroPoint,
bits: 8,
data_type: DataType::F16,
group_size: 64,