A high-performance inference engine for AI models
Optimize and unify low-batch matmul (#1023)
CCYehece61cb866bf
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rs b/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rsindex 0e28f0315..0f086ae56 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rs@@ -5,4 +5,5 @@ pub enum ActivationTransformOp {OutputRht,Quantize,QuantizeWithGroupSums,+ PackHalf,}diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rsindex d4605fbdb..300310824 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rs@@ -4,4 +4,5 @@ pub enum GatedActMulOp {FullPrecision,Quantize,QuantizeWithGroupSums,+ PackHalf,}diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rsindex 73018d002..d5ff4cc24 100644--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs@@ -2,7 +2,10 @@ use crate::{backends::common::{Backend, BufferMut, BufferRef, CommandBuffer, Kernels,gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},- kernel::{ActivationTransformKernel, matmul::Int8CodeLayout},+ kernel::{+ ActivationTransformKernel,+ matmul::{ActivationFormat, Int8CodeLayout},+ },},data_type::DataType,};@@ -16,7 +19,7 @@ fn assert_row_width(element_count: u32) {pub const ACTIVATION_SCALE_GROUP_SIZE: u32 = 128;-#[derive(Clone, Copy)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub struct ActivationQuantization {scale_group_size: u32,sum_group_size: Option<u32>,@@ -55,7 +58,7 @@ pub struct ActivationTransform<B: Backend> {kernel: <B::Kernels as Kernels>::ActivationTransformKernel,in_place: bool,has_bias: bool,- quantization: Option<ActivationQuantization>,+ format: ActivationFormat,}impl<B: Backend> ActivationTransform<B> {@@ -65,12 +68,21 @@ impl<B: Backend> ActivationTransform<B> {bias_data_type: Option<DataType>,ops: ActivationTransformOp,in_place: bool,- quantization: Option<ActivationQuantization>,+ format: ActivationFormat,) -> Result<Self, B::Error> {let has_bias = bias_data_type.is_some();- let (codes_grouped_by_nibble, scale_group_size, sum_group_size) = match quantization {- Some(q) => (q.code_layout.is_grouped_by_nibble(), q.scale_group_size, q.sum_group_size),- None => (false, HADAMARD_TRANSFORM_BLOCK_SIZE, None),+ let (codes_grouped_by_nibble, scale_group_size, sum_group_size) = match format {+ ActivationFormat::Int8(q) => (+ q.code_layout().is_grouped_by_nibble(),+ q.scale_group_size(),+ q.sum_group_size().unwrap_or(q.scale_group_size()),+ ),+ ActivationFormat::PackedHalf {+ group_size,+ } => (false, HADAMARD_TRANSFORM_BLOCK_SIZE, group_size),+ ActivationFormat::Bf16 | ActivationFormat::Trellis => {+ (false, HADAMARD_TRANSFORM_BLOCK_SIZE, HADAMARD_TRANSFORM_BLOCK_SIZE)+ },};let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(context,@@ -80,14 +92,14 @@ impl<B: Backend> ActivationTransform<B> {codes_grouped_by_nibble,in_place,scale_group_size,- sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE),+ sum_group_size,has_bias,)?;Ok(Self {kernel,in_place,has_bias,- quantization,+ format,})}@@ -96,7 +108,7 @@ impl<B: Backend> ActivationTransform<B> {data_type: DataType,in_place: bool,) -> Result<Self, B::Error> {- Self::new(context, data_type, None, ActivationTransformOp::InputRht, in_place, None)+ Self::new(context, data_type, None, ActivationTransformOp::InputRht, in_place, ActivationFormat::Bf16)}pub fn output_rht(@@ -105,30 +117,34 @@ impl<B: Backend> ActivationTransform<B> {bias_data_type: Option<DataType>,in_place: bool,) -> Result<Self, B::Error> {- Self::new(context, data_type, bias_data_type, ActivationTransformOp::OutputRht, in_place, None)+ Self::new(+ context,+ data_type,+ bias_data_type,+ ActivationTransformOp::OutputRht,+ in_place,+ ActivationFormat::Bf16,+ )}pub fn quantize(context: &B::Context,data_type: DataType,- quantization: ActivationQuantization,+ quantization: ActivationFormat,) -> Result<Self, B::Error> {- let op = quantization- .sum_group_size()- .map_or(ActivationTransformOp::Quantize, |_| ActivationTransformOp::QuantizeWithGroupSums);- Self::new(context, data_type, None, op, false, Some(quantization))- }-- pub fn scale_group_size(&self) -> u32 {- self.quantization.expect("quantized activation transform required").scale_group_size()- }-- pub fn sum_group_size(&self) -> Option<u32> {- self.quantization.expect("quantized activation transform required").sum_group_size()+ let op = match quantization {+ ActivationFormat::Int8(q) if q.sum_group_size().is_some() => ActivationTransformOp::QuantizeWithGroupSums,+ ActivationFormat::Int8(_) => ActivationTransformOp::Quantize,+ ActivationFormat::PackedHalf {+ ..+ } => ActivationTransformOp::PackHalf,+ ActivationFormat::Bf16 | ActivationFormat::Trellis => unreachable!(),+ };+ Self::new(context, data_type, None, op, false, quantization)}- pub fn code_layout(&self) -> Int8CodeLayout {- self.quantization.expect("quantized activation transform required").code_layout()+ pub fn format(&self) -> ActivationFormat {+ self.format}/// `input` and `output` must be distinct buffers.@@ -141,7 +157,7 @@ impl<B: Backend> ActivationTransform<B> {element_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {- assert!(self.quantization.is_none() && !self.in_place && !self.has_bias);+ assert!(self.format == ActivationFormat::Bf16 && !self.in_place && !self.has_bias);assert_row_width(element_count);self.kernel.encode(Some(input),@@ -153,6 +169,7 @@ impl<B: Backend> ActivationTransform<B> {rht_factors,batch_size,element_count,+ None::<&mut B::ScratchBuffer>,command_buffer,);}@@ -166,7 +183,7 @@ impl<B: Backend> ActivationTransform<B> {element_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {- assert!(self.quantization.is_none() && self.in_place && self.has_bias == bias.is_some());+ assert!(self.format == ActivationFormat::Bf16 && self.in_place && self.has_bias == bias.is_some());assert_row_width(element_count);self.kernel.encode(None::<&B::ScratchBuffer>,@@ -178,6 +195,7 @@ impl<B: Backend> ActivationTransform<B> {rht_factors,batch_size,element_count,+ None::<&mut B::ScratchBuffer>,command_buffer,);}@@ -193,7 +211,9 @@ impl<B: Backend> ActivationTransform<B> {element_count: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {- let quantization = self.quantization.expect("quantized activation transform required");+ let ActivationFormat::Int8(quantization) = self.format else {+ panic!("quantized activation transform required")+ };assert_row_width(element_count);assert!(element_count.is_multiple_of(quantization.scale_group_size()),@@ -213,6 +233,33 @@ impl<B: Backend> ActivationTransform<B> {rht_factors,batch_size,element_count,+ None::<&mut B::ScratchBuffer>,+ command_buffer,+ );+ }++ pub fn encode_pack_half(+ &self,+ input: impl BufferRef<Backend = B>,+ output: impl BufferMut<Backend = B>,+ rht_factors: impl BufferRef<Backend = B>,+ batch_size: u32,+ element_count: u32,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) {+ assert!(matches!(self.format, ActivationFormat::PackedHalf { .. }) && batch_size <= 8);+ assert_row_width(element_count);+ self.kernel.encode(+ Some(input),+ None::<&mut B::ScratchBuffer>,+ None::<&B::GlobalGpuBuffer>,+ None::<&mut B::ScratchBuffer>,+ None::<&mut B::ScratchBuffer>,+ None::<&mut B::ScratchBuffer>,+ rht_factors,+ batch_size,+ element_count,+ Some(output),command_buffer,);}diff --git a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rsindex 77dc2bd4d..9a4a6610a 100644--- a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs@@ -7,7 +7,7 @@ use crate::{activation_type::ActivationType, gated_act_mul::GatedActMulOp,hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,},- kernel::{ActivationQuantization, GatedActMulKernel},+ kernel::{GatedActMulKernel, matmul::ActivationFormat},},config::clipping::ClippingBounds,data_type::DataType,@@ -33,7 +33,7 @@ pub struct GatedActMul<B: Backend> {kernel: <B::Kernels as Kernels>::GatedActMulKernel,options: GatedActMulOptions,settings: GatedActMulSettings,- quantization: Option<ActivationQuantization>,+ format: ActivationFormat,}impl<B: Backend> GatedActMul<B> {@@ -47,17 +47,17 @@ impl<B: Backend> GatedActMul<B> {let mut options = GatedActMulOptions::empty();options.set(GatedActMulOptions::INTERLEAVED, interleaved);options.set(GatedActMulOptions::HADAMARD, use_hadamard);- Self::new(context, data_type, options, settings, None)+ Self::new(context, data_type, options, settings, ActivationFormat::Bf16)}pub fn quantized(context: &B::Context,data_type: DataType,- quantization: ActivationQuantization,+ quantization: ActivationFormat,settings: GatedActMulSettings,) -> Result<Self, B::Error> {let options = GatedActMulOptions::INTERLEAVED | GatedActMulOptions::HADAMARD;- Self::new(context, data_type, options, settings, Some(quantization))+ Self::new(context, data_type, options, settings, quantization)}fn new(@@ -65,10 +65,10 @@ impl<B: Backend> GatedActMul<B> {data_type: DataType,options: GatedActMulOptions,settings: GatedActMulSettings,- quantization: Option<ActivationQuantization>,+ format: ActivationFormat,) -> Result<Self, B::Error> {- let (ops, codes_grouped_by_nibble, scale_group_size, sum_group_size) = match quantization {- Some(quantization) => (+ let (ops, codes_grouped_by_nibble, scale_group_size, sum_group_size) = match format {+ ActivationFormat::Int8(quantization) => (if quantization.sum_group_size().is_some() {GatedActMulOp::QuantizeWithGroupSums} else {@@ -78,7 +78,12 @@ impl<B: Backend> GatedActMul<B> {quantization.scale_group_size(),quantization.sum_group_size().unwrap_or(quantization.scale_group_size()),),- None => (GatedActMulOp::FullPrecision, false, HADAMARD_TRANSFORM_BLOCK_SIZE, HADAMARD_TRANSFORM_BLOCK_SIZE),+ ActivationFormat::PackedHalf {+ group_size,+ } => (GatedActMulOp::PackHalf, false, HADAMARD_TRANSFORM_BLOCK_SIZE, group_size),+ ActivationFormat::Bf16 | ActivationFormat::Trellis => {+ (GatedActMulOp::FullPrecision, false, HADAMARD_TRANSFORM_BLOCK_SIZE, HADAMARD_TRANSFORM_BLOCK_SIZE)+ },};let kernel = <B::Kernels as Kernels>::GatedActMulKernel::new(context,@@ -97,10 +102,14 @@ impl<B: Backend> GatedActMul<B> {kernel,options,settings,- quantization,+ format,})}+ pub fn format(&self) -> ActivationFormat {+ self.format+ }+pub fn encode_fp(&self,act_operand: impl BufferRef<Backend = B>,@@ -114,7 +123,7 @@ impl<B: Backend> GatedActMul<B> {act_type: ActivationType,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {- assert!(self.quantization.is_none());+ assert_eq!(self.format, ActivationFormat::Bf16);assert_eq!(self.options.contains(GatedActMulOptions::INTERLEAVED), value_operand.is_none());assert_eq!(self.options.contains(GatedActMulOptions::HADAMARD), hadamard_factors.is_some());assert!(@@ -141,6 +150,7 @@ impl<B: Backend> GatedActMul<B> {gate_clip_max,value_clip_min,value_clip_max,+ None::<&mut B::ScratchBuffer>,command_buffer,);}@@ -157,7 +167,9 @@ impl<B: Backend> GatedActMul<B> {act_type: ActivationType,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {- let quantization = self.quantization.expect("quantized gated activation required");+ let ActivationFormat::Int8(quantization) = self.format else {+ panic!("quantized gated activation required")+ };assert!(self.options.contains(GatedActMulOptions::INTERLEAVED));assert!(self.options.contains(GatedActMulOptions::HADAMARD));assert!(gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));@@ -186,6 +198,44 @@ impl<B: Backend> GatedActMul<B> {gate_clip_max,value_clip_min,value_clip_max,+ None::<&mut B::ScratchBuffer>,+ command_buffer,+ );+ }++ pub fn encode_pack_half(+ &self,+ act_operand: impl BufferRef<Backend = B>,+ output: impl BufferMut<Backend = B>,+ hadamard_factors: impl BufferRef<Backend = B>,+ gated_dim: u32,+ batch_dim: u32,+ act_type: ActivationType,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) {+ assert!(matches!(self.format, ActivationFormat::PackedHalf { .. }) && batch_dim <= 8);+ assert!(gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));+ let (gate_clip_min, gate_clip_max) = self.settings.gate_clipping.into_pair().unzip();+ let (value_clip_min, value_clip_max) = self.settings.value_clipping.into_pair().unzip();+ self.kernel.encode(+ act_operand,+ None::<&B::ScratchBuffer>,+ None::<&mut B::ScratchBuffer>,+ None::<&mut B::ScratchBuffer>,+ None::<&mut B::ScratchBuffer>,+ None::<&mut B::ScratchBuffer>,+ Some(hadamard_factors),+ gated_dim,+ batch_dim,+ 0,+ 0,+ act_type,+ self.settings.activation_alpha,+ gate_clip_min,+ gate_clip_max,+ value_clip_min,+ value_clip_max,+ Some(output),command_buffer,);}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rsindex 717da5b7c..030cd9494 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs@@ -47,10 +47,20 @@ pub trait MatmulKernel: Sized + Send + Sync {None}+ fn activation_preparation(+ &self,+ _shape: &MatmulShape,+ _weights_layout: Self::WeightsLayout,+ _context: &<Self::Backend as Backend>::Context,+ ) -> ActivationFormat {+ ActivationFormat::Bf16+ }+fn select_activation_format(&self,_bf16_shape: &MatmulShape,_weights_layout: Self::WeightsLayout,+ _prepared: ActivationFormat,_context: &<Self::Backend as Backend>::Context,) -> ActivationFormat {ActivationFormat::Bf16diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rsindex 4b3473809..dc69c1203 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs@@ -1,8 +1,8 @@use std::mem::size_of;-use crate::backends::common::BufferRef;+use crate::backends::common::kernel::ActivationQuantization;-#[derive(Clone, Copy, PartialEq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum Int8CodeLayout {Sequential,GroupedByNibble,@@ -46,17 +46,21 @@ impl Int8CodeLayout {}}-pub enum MatmulA<T: BufferRef> {+#[derive(Clone, Copy)]+pub enum MatmulA<T> {FullPrecision {values: T,offset: usize,},+ PackedHalf {+ values: T,+ group_size: u32,+ },Int8Symmetric {values: T,scales: T,group_sums: Option<T>,- scale_group_size: u32,- code_layout: Int8CodeLayout,+ quantization: ActivationQuantization,},Trellis {values: T,@@ -64,3 +68,60 @@ pub enum MatmulA<T: BufferRef> {scales: T,},}++impl<T> MatmulA<T> {+ pub fn full_precision(values: T) -> Self {+ Self::FullPrecision {+ values,+ offset: 0,+ }+ }++ pub fn as_ref(&self) -> MatmulA<&T> {+ match self {+ Self::FullPrecision {+ values,+ offset,+ } => MatmulA::FullPrecision {+ values,+ offset: *offset,+ },+ Self::PackedHalf {+ values,+ group_size,+ } => MatmulA::PackedHalf {+ values,+ group_size: *group_size,+ },+ Self::Int8Symmetric {+ values,+ scales,+ group_sums,+ quantization,+ } => MatmulA::Int8Symmetric {+ values,+ scales,+ group_sums: group_sums.as_ref(),+ quantization: *quantization,+ },+ Self::Trellis {+ values,+ column_group_sums,+ scales,+ } => MatmulA::Trellis {+ values,+ column_group_sums,+ scales,+ },+ }+ }+}++/// Matches `PackedHalfLayout` in `packed_half_layout.h`.+pub fn packed_half_bytes(+ k: u32,+ group_size: u32,+) -> usize {+ assert!(matches!(group_size, 32 | 64) && k.is_multiple_of(group_size));+ (k / group_size) as usize * (32 * (group_size as usize / 8) + 16) * size_of::<u32>()+}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rsindex d70ba1475..1d2030d4a 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs@@ -9,7 +9,7 @@ pub mod routing;pub use arguments::MatmulArguments;pub use d_ops::MatmulDOps;pub use kernel::MatmulKernel;-pub use matmul_a::{Int8CodeLayout, MatmulA};+pub use matmul_a::{Int8CodeLayout, MatmulA, packed_half_bytes};pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};#[cfg(backend = "metal")]pub use matmul_b::{TRELLIS_CODES_TAIL_BYTES, TrellisFormat};diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rsindex 98d2df91d..83518b514 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs@@ -2,12 +2,17 @@ use super::{MatmulA, MatmulArguments, QuantParamsLayout};use crate::backends::common::{Backend, BufferMut, BufferRef,gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ kernel::ActivationQuantization,};#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum ActivationFormat {Bf16,- Int8,+ Int8(ActivationQuantization),+ PackedHalf {+ group_size: u32,+ },+ Trellis,}#[derive(Clone, Copy)]@@ -22,7 +27,7 @@ pub struct MatmulShape {pub b_bits: Option<u32>,pub b_group_size: Option<u32>,pub signed_codes: bool,- pub a_full_precision: bool,+ pub a: ActivationFormat,pub gathered: bool,pub params_layout: Option<QuantParamsLayout>,pub d_transform: GemmDTransform,@@ -50,7 +55,24 @@ impl MatmulShape {b_bits: arguments.b.bits_per_b(),b_group_size: arguments.b.group_size(),signed_codes: arguments.b.signed_codes(),- a_full_precision: matches!(arguments.a, MatmulA::FullPrecision { .. }),+ a: match &arguments.a {+ MatmulA::FullPrecision {+ ..+ } => ActivationFormat::Bf16,+ MatmulA::PackedHalf {+ group_size,+ ..+ } => ActivationFormat::PackedHalf {+ group_size: *group_size,+ },+ MatmulA::Int8Symmetric {+ quantization,+ ..+ } => ActivationFormat::Int8(*quantization),+ MatmulA::Trellis {+ ..+ } => ActivationFormat::Trellis,+ },gathered: arguments.gather_indices.is_some(),params_layout: arguments.b.quant_params_layout(),d_transform: arguments.output.ops.mask(),@@ -60,4 +82,8 @@ impl MatmulShape {pub fn is_quant(&self) -> bool {self.b_prologue != GemmBPrologueKind::FullPrecision}++ pub fn a_is_quantized(&self) -> bool {+ matches!(self.a, ActivationFormat::Int8(_) | ActivationFormat::Trellis)+ }}diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rsindex 034b6672c..5a3a3efaf 100644--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs@@ -19,6 +19,7 @@ pub use attention::{AttentionArguments, AttentionInput, AttentionKernel, AttentionKernelConfig, FullPrecisionInput, Int8Input, KVFormat,};pub use gated_act_mul::{GatedActMul, GatedActMulSettings};+pub use matmul::ActivationFormat;#[cfg(any(backend = "cpu", test))]pub use crate::backends::common::kernel::trellis::mixing_dimension;diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rsindex 2a7e1bb54..a501ae16b 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs@@ -64,6 +64,7 @@ pub fn activation_transform<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Floatrht_factors: *const i32,batch_size: u32,element_count: u32,+ #[optional(ops == ActivationTransformOp::PackHalf)] packed_half_out: Option<*mut u32>,#[specialize] ops: ActivationTransformOp,#[specialize] grouped_by_weight_nibble: bool,#[specialize] in_place: bool,@@ -79,6 +80,7 @@ pub fn activation_transform<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Floatlet columns = element_count as usize;let input_rht = ops != ActivationTransformOp::OutputRht;let quantize = matches!(ops, ActivationTransformOp::Quantize | ActivationTransformOp::QuantizeWithGroupSums);+ assert!(packed_half_out.is_none(), "CPU packed-half activation is not selected");assert_eq!(bias.is_some(), has_bias);let mut transformed = vec![0.0f32; columns];diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rsindex 95256b52c..15f41254a 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs@@ -41,6 +41,7 @@ pub fn gated_act_mul<T: DataTypeRepr + Float>(#[optional(clip_gate)] gate_clip_max: Option<f32>,#[optional(clip_value)] value_clip_min: Option<f32>,#[optional(clip_value)] value_clip_max: Option<f32>,+ #[optional(ops == GatedActMulOp::PackHalf)] packed_half_out: Option<*mut u32>,#[specialize] ops: GatedActMulOp,#[specialize] grouped_by_weight_nibble: bool,#[specialize] interleaved: bool,@@ -54,6 +55,7 @@ pub fn gated_act_mul<T: DataTypeRepr + Float>(assert_eq!(hadamard_factors.is_some(), use_hadamard);let quantize = matches!(ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums);let emit_group_sums = ops == GatedActMulOp::QuantizeWithGroupSums;+ assert!(packed_half_out.is_none(), "CPU packed-half gate is not selected");assert!(!quantize || use_hadamard, "quantized gate activation requires RHT");assert!(!quantize || gated_dim.is_multiple_of(activation_scale_group_size));assert!(!emit_group_sums || gated_dim.is_multiple_of(sum_group_size));diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex 94debb345..7716791dc 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -122,9 +122,10 @@ impl MatmulKernel for MatmulCpuKernel {values,scales,group_sums: _,- scale_group_size: a_group_size,- code_layout,+ quantization,} => {+ let a_group_size = quantization.scale_group_size();+ let code_layout = quantization.code_layout();let compatible = matches!(a_group_size, 32 | 64 | 128)&& k.is_multiple_of(a_group_size)&& matches!(b.group_size(), Some(32 | 64 | 128))@@ -149,7 +150,10 @@ impl MatmulKernel for MatmulCpuKernel {code_layout,}},- MatmulA::Trellis {+ MatmulA::PackedHalf {+ ..+ }+ | MatmulA::Trellis {..} => {return Err(MatmulCpuError::UnsupportedLayout {diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rsindex 57c2b1784..35c8a2873 100644--- a/crates/uzu-engine/src/backends/metal/context.rs+++ b/crates/uzu-engine/src/backends/metal/context.rs@@ -34,8 +34,6 @@ use crate::backends::{},};-pub(super) const LARGE_MIN_GPU_CORES: u32 = 30;-pub(super) struct SparseState {pub did_ops: bool,pub event_value: u64,diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rsindex c42e4d23c..4c51fa007 100644--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs@@ -6,6 +6,7 @@ use derive_more::Display;pub enum GemmAPrologueKind {FullPrecision,Int8Symmetric,+ PackedHalf,}#[repr(C)]@@ -32,11 +33,17 @@ pub enum GemmTiling {Tile64x32x256_Simdgroups4x1,Tile64x64x256_Simdgroups2x2,Tile128x128x256_Simdgroups4x4,+ Tile8x32x64_Simdgroups1x2x4,+ Tile8x32x128_Simdgroups1x2x4,}pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;impl GemmTiling {+ pub const fn is_packed_w4(self) -> bool {+ matches!(self, Self::Tile8x32x64_Simdgroups1x2x4)+ }+pub const fn block_m(self) -> u32 {match self {Self::Tile8x32x32_Simdgroups1x1 => 8,@@ -51,6 +58,7 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 64,Self::Tile64x64x256_Simdgroups2x2 => 64,Self::Tile128x128x256_Simdgroups4x4 => 128,+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 8,}}@@ -68,6 +76,7 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 32,Self::Tile64x64x256_Simdgroups2x2 => 64,Self::Tile128x128x256_Simdgroups4x4 => 128,+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 32,}}@@ -85,6 +94,8 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 256,Self::Tile64x64x256_Simdgroups2x2 => 256,Self::Tile128x128x256_Simdgroups4x4 => 256,+ Self::Tile8x32x64_Simdgroups1x2x4 => 64,+ Self::Tile8x32x128_Simdgroups1x2x4 => 128,}}@@ -102,6 +113,7 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 4,Self::Tile64x64x256_Simdgroups2x2 => 2,Self::Tile128x128x256_Simdgroups4x4 => 4,+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 1,}}@@ -119,12 +131,15 @@ impl GemmTiling {Self::Tile64x32x256_Simdgroups4x1 => 1,Self::Tile64x64x256_Simdgroups2x2 => 2,Self::Tile128x128x256_Simdgroups4x4 => 4,+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 2,}}pub const fn simdgroups_k(self) -> u32 {match self {- Self::Tile16x32x256_Simdgroups1x1x4 => 4,+ Self::Tile16x32x256_Simdgroups1x1x4+ | Self::Tile8x32x64_Simdgroups1x2x4+ | Self::Tile8x32x128_Simdgroups1x2x4 => 4,_ => 1,}}@@ -141,6 +156,10 @@ impl GemmTiling {| Self::Tile128x128x256_Simdgroups4x4)}++ pub const fn is_small_batch_variant(self) -> bool {+ matches!(self, Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4)+ }}pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metalindex da31d80d2..2dc3e562d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include "../common/activation_quantization.h"#include "../common/defines.h"+#include "../common/packed_half_activation.h"#include "../common/conversion.h"#include "../common/dsl.h"#include "../common/thread_context.h"@@ -16,13 +17,14 @@ using namespace uzu::activation_transform;#define QUANTIZED (ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)#define EMITS_GROUP_SUMS (ops == ActivationTransformOp::QuantizeWithGroupSums)+#define PACKED_HALF (ops == ActivationTransformOp::PackHalf)template <typename T, typename BiasT>VARIANTS(T, float, bfloat)VARIANTS(BiasT, float, bfloat)PUBLIC KERNEL(ActivationTransform)(const device T* input OPTIONAL(!in_place),- device T* fp_out OPTIONAL(!QUANTIZED),+ device T* fp_out OPTIONAL(!QUANTIZED && !PACKED_HALF),const device BiasT* bias OPTIONAL(has_bias),device int8_t* q_out OPTIONAL(QUANTIZED),device float* scales_out OPTIONAL(QUANTIZED),@@ -30,6 +32,7 @@ PUBLIC KERNEL(ActivationTransform)(const device int32_t* rht_factors,constant uint& batch_size,constant uint& element_count,+ device uint* packed_half_out OPTIONAL(PACKED_HALF),const ActivationTransformOp ops SPECIALIZE,const bool grouped_by_weight_nibble SPECIALIZE,const bool in_place SPECIALIZE,@@ -65,7 +68,7 @@ PUBLIC KERNEL(ActivationTransform)(values *= float4(factors);}- if (!QUANTIZED) {+ if (!QUANTIZED && !PACKED_HALF) {if (in_bounds) {if (has_bias) {if constexpr (metal::is_same_v<T, bfloat>) {@@ -78,6 +81,13 @@ PUBLIC KERNEL(ActivationTransform)(return;}+ if (PACKED_HALF) {+ if (in_bounds) {+ store_packed_half(values, lane_index, first_index, batch_index, sum_group_size, packed_half_out);+ }+ return;+ }+store_quantized_activation_vector(values,lane_index,diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/packed_half_activation.h b/crates/uzu-engine/src/backends/metal/kernel/common/packed_half_activation.hnew file mode 100644index 000000000..f60fa6407--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/common/packed_half_activation.h@@ -0,0 +1,44 @@+#pragma once++#include <metal_stdlib>++#include "activation_quantization.h"+#include "../matmul/common/packed_half_layout.h"+#include "../matmul/common/quant_unpack.h"++using namespace metal;++UZU_CONST float PACKED_HALF_MAX_FINITE = 65504.0f;++static METAL_FUNC void store_packed_half(+ const float4 values,+ const uint lane,+ const uint first_index,+ const uint row,+ const uint group_size,+ device uint* packed+) {+ using Layout32 = uzu::gemm::PackedHalfLayout<32>;+ using Layout64 = uzu::gemm::PackedHalfLayout<64>;+ const uint group = first_index / group_size;+ const uint k = first_index % group_size;+ const uint group_words = group_size == 32 ? Layout32::GROUP_WORDS : Layout64::GROUP_WORDS;+ const uint word =+ group * group_words + (group_size == 32 ? Layout32::word_offset(k, row) : Layout64::word_offset(k, row));+ device half* halves = reinterpret_cast<device half*>(packed);+ float2 sums = 0.0f;+ for (uint i = 0; i < Layout32::VALUES_PER_VECTOR; ++i) {+ const half value = half(clamp(values[i], -PACKED_HALF_MAX_FINITE, PACKED_HALF_MAX_FINITE));+ halves[Layout32::ROWS_PER_PAIR * (word + i) + row % Layout32::ROWS_PER_PAIR] = value;+ sums += float2(float(value), uzu::gemm::w4_pair_bias_value(i) * float(value));+ }+ const uint lane_count = group_size / Layout32::VALUES_PER_VECTOR;+ sums.x = reduce_adjacent_lanes(sums.x, lane_count, [](float x, float y) { return x + y; });+ sums.y = reduce_adjacent_lanes(sums.y, lane_count, [](float x, float y) { return x + y; });+ if (lane % lane_count == 0) {+ const uint sums_offset = group_size == 32 ? Layout32::sums_offset() : Layout64::sums_offset();+ device float* output = reinterpret_cast<device float*>(packed + group * group_words + sums_offset);+ output[row * Layout32::SUMS_PER_ROW] = sums.x;+ output[row * Layout32::SUMS_PER_ROW + 1] = sums.y;+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metalindex 85b32e5f4..f3ae5f572 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal@@ -2,6 +2,7 @@#include "../common/activation_quantization.h"#include "../common/dsl.h"#include "../common/gated_act_mul.h"+#include "../common/packed_half_activation.h"#include "../common/thread_context.h"#include "../generated/gated_act_mul.h"#include "../hadamard_transform/hadamard_transform.h"@@ -16,13 +17,14 @@ using namespace uzu::gated_act_mul;#define QUANTIZED (ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)#define EMITS_GROUP_SUMS (ops == GatedActMulOp::QuantizeWithGroupSums)+#define PACKED_HALF (ops == GatedActMulOp::PackHalf)template <typename T>VARIANTS(T, float, bfloat)PUBLIC KERNEL(GatedActMul) (const device T* act_operand,const device T* value_operand OPTIONAL(!interleaved),- device T* fp_out OPTIONAL(!QUANTIZED),+ device T* fp_out OPTIONAL(!QUANTIZED && !PACKED_HALF),device int8_t* q_out OPTIONAL(QUANTIZED),device float* scales_out OPTIONAL(QUANTIZED),device int32_t* group_sums_out OPTIONAL(EMITS_GROUP_SUMS),@@ -37,6 +39,7 @@ PUBLIC KERNEL(GatedActMul) (const constant float& gate_clip_max OPTIONAL(clip_gate),const constant float& value_clip_min OPTIONAL(clip_value),const constant float& value_clip_max OPTIONAL(clip_value),+ device uint* packed_half_out OPTIONAL(PACKED_HALF),const GatedActMulOp ops SPECIALIZE,const bool grouped_by_weight_nibble SPECIALIZE,const bool interleaved SPECIALIZE,@@ -109,7 +112,7 @@ PUBLIC KERNEL(GatedActMul) (results = simdgroup_hadamard_transform_vector(lane_index, results);}- if (!QUANTIZED) {+ if (!QUANTIZED && !PACKED_HALF) {device T* out_row = fp_out + batch_idx * gated_dim;if (use_vector_io) {store_hadamard_vector(out_row + first_index, results);@@ -121,6 +124,13 @@ PUBLIC KERNEL(GatedActMul) (return;}+ if (PACKED_HALF) {+ if (valid_count > 0) {+ store_packed_half(results, lane_index, first_index, batch_idx, sum_group_size, packed_half_out);+ }+ return;+ }+store_quantized_activation_vector(results,lane_index,diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.h b/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.hindex 70797b2f0..5f057e526 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.h@@ -10,5 +10,6 @@ enum class ActivationTransformOp : uint32_t {OutputRht = 1,Quantize = 2,QuantizeWithGroupSums = 3,+ PackHalf = 4,};} // namespace uzu::activation_transformdiff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.hindex 571e56256..307a6d667 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.h@@ -9,5 +9,6 @@ enum class GatedActMulOp : uint32_t {FullPrecision = 0,Quantize = 1,QuantizeWithGroupSums = 2,+ PackHalf = 3,};} // namespace uzu::gated_act_muldiff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.hindex b555f029c..34d1c9dda 100644--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h@@ -29,6 +29,7 @@ struct GemmDTransform {enum class GemmAPrologueKind : uint32_t {FullPrecision = 0,Int8Symmetric = 1,+ PackedHalf = 2,};enum class GemmBLayout : uint32_t {@@ -49,6 +50,8 @@ enum class GemmTiling : uint32_t {Tile64x32x256_Simdgroups4x1 = 9,Tile64x64x256_Simdgroups2x2 = 10,Tile128x128x256_Simdgroups4x4 = 11,+ Tile8x32x64_Simdgroups1x2x4 = 12,+ Tile8x32x128_Simdgroups1x2x4 = 13,};static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.hindex 2bf74883f..0a939a0f2 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h@@ -303,20 +303,22 @@ struct Fragment {bool ALIGNED_N = false,class Ptr,class FragmentRowStride = Int<1>,- class FragmentColStride = Int<1>>+ class FragmentColStride = Int<1>,+ class ColStride = Int<1>>METAL_FUNC void store_safe(const ushort simd_lane_id,Ptr destination,- const int leading_dimension,+ const int row_stride,const short2 tile_dimensions,FragmentRowStride fragment_row_stride = {},- FragmentColStride fragment_col_stride = {}+ FragmentColStride fragment_col_stride = {},+ ColStride col_stride = {}) thread {transfer<STORE, SAFE, ALIGNED_M, ALIGNED_N>(simd_lane_id,destination,- leading_dimension,- Int<1>{},+ row_stride,+ col_stride,tile_dimensions.y,tile_dimensions.x,fragment_row_stride,@@ -360,6 +362,38 @@ struct Fragment {}}+ template <ushort K_SLICES, ushort OUTPUT_GROUPS>+ METAL_FUNC bool reduce_k_slices(+ threadgroup ElementType* partials,+ const thread ThreadContext& thread_context+ ) thread {+ if constexpr (K_SLICES == 1) {+ return true;+ } else {+ const ushort k_slice = ushort(thread_context.simdgroup_index) / OUTPUT_GROUPS;+ const ushort output_group = ushort(thread_context.simdgroup_index) % OUTPUT_GROUPS;+ const ushort lane = ushort(thread_context.simd_lane_id);+ thread ElementType* values = elements();+ constexpr ushort ELEMENTS = ELEMENTS_PER_FRAGMENT;+ constexpr ushort GROUP_STRIDE = (K_SLICES - 1) * ELEMENTS * METAL_SIMD_SIZE;+ if (k_slice != 0) {+ for (ushort e = 0; e < ELEMENTS; ++e) {+ partials[output_group * GROUP_STRIDE + ((k_slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane] = values[e];+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (k_slice != 0) {+ return false;+ }+ for (ushort slice = 1; slice < K_SLICES; ++slice) {+ for (ushort e = 0; e < ELEMENTS; ++e) {+ values[e] += partials[output_group * GROUP_STRIDE + ((slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane];+ }+ }+ return true;+ }+ }+private:UZU_CONST bool LOAD = true;UZU_CONST bool STORE = false;@@ -435,6 +469,21 @@ private:}};+template <typename T, ushort ROWS, ushort COLS, class Ops, class Generator>+struct GeneratedFragment {+ using ElementType = T;+ using FragmentOpsType = Ops;+ using ThreadVectorType = typename Ops::template ThreadVector<T>;+ UZU_CONST ushort ROW_FRAGMENTS = ROWS;+ UZU_CONST ushort COL_FRAGMENTS = COLS;+ UZU_CONST bool MMA_TRANSPOSE = false;+ Generator generator;++ METAL_FUNC ThreadVectorType fragment_at(const ushort row, const ushort col) const thread {+ return generator(row, col);+ }+};+template <class Ops, class Read>struct OperandFragmentTraits {UZU_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/packed_half_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/packed_half_layout.hnew file mode 100644index 000000000..40fc02d49--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/packed_half_layout.h@@ -0,0 +1,36 @@+#pragma once++#include <metal_stdlib>+#include "../../common/defines.h"++using namespace metal;++namespace uzu::gemm {++template <uint GROUP_SIZE>+struct PackedHalfLayout {+ UZU_CONST uint MMA_WIDTH = 8;+ UZU_CONST uint VALUES_PER_VECTOR = 4;+ UZU_CONST uint ROWS_PER_PAIR = 2;+ UZU_CONST uint SUMS_PER_ROW = 2;+ UZU_CONST uint G64_K_INTERLEAVE = 2;+ UZU_CONST uint G64_K_CHUNK = G64_K_INTERLEAVE * MMA_WIDTH;+ UZU_CONST uint STEPS = GROUP_SIZE / MMA_WIDTH;+ UZU_CONST uint GROUP_WORDS = METAL_SIMD_SIZE * STEPS + (METAL_SIMD_SIZE / MMA_WIDTH) * ROWS_PER_PAIR * SUMS_PER_ROW;++ static METAL_FUNC uint pair_offset(uint slot, uint fm) { return (slot * MMA_WIDTH + fm) * STEPS; }+ static METAL_FUNC uint codes_offset(uint group, uint slot) {+ return group * (GROUP_SIZE / 2) + slot * (GROUP_SIZE / 8);+ }+ static METAL_FUNC uint sums_offset() { return METAL_SIMD_SIZE * STEPS; }+ static METAL_FUNC uint word_offset(uint k, uint row) {+ const uint chunk = (k % G64_K_CHUNK) / VALUES_PER_VECTOR;+ const uint fm =+ GROUP_SIZE == 32 ? k / VALUES_PER_VECTOR : (k / G64_K_CHUNK) * G64_K_INTERLEAVE + chunk % G64_K_INTERLEAVE;+ const uint step = GROUP_SIZE == 32 ? k % VALUES_PER_VECTOR+ : (chunk / G64_K_INTERLEAVE) * VALUES_PER_VECTOR + k % VALUES_PER_VECTOR;+ return pair_offset(row / ROWS_PER_PAIR, fm) + step;+ }+};++} // namespace uzu::gemmdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.hindex 4524c4d93..ffe4ce189 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -31,6 +31,58 @@ UZU_CONST ushort W4_BITS = 4;UZU_CONST uint W4_SIGN_MASK = symmetric_zero_point<W4_BITS>() * 0x11111111u;UZU_CONST uint W8_SIGN_MASK = symmetric_zero_point<8>() * 0x01010101u;UZU_CONST uint W4_NIBBLE_MASK = 0x0F0F0F0Fu;+UZU_CONST uint ZERO_POINTS_PER_BYTE = 2;+UZU_CONST uint W8_MAGIC_FLOAT_OFFSET = 0x4B400000u;++UZU_CONST uint W4_PAIR_HALF_BITS = 16;+UZU_CONST uint W4_PAIR_MANTISSA_BITS = 10;+UZU_CONST uint W4_PAIR_EXPONENT_BIAS = 15;++static METAL_FUNC constexpr uint w4_pair_offset(uint component) {+ constexpr uint COMPONENTS = W4_PAIR_HALF_BITS / W4_BITS;+ return W4_BITS * (component < COMPONENTS ? component : COMPONENTS - 1);+}++static METAL_FUNC constexpr uint w4_pair_position(uint component) {+ const uint offset = w4_pair_offset(component);+ return offset + W4_BITS <= W4_PAIR_MANTISSA_BITS ? offset : offset - (W4_PAIR_HALF_BITS - W4_PAIR_MANTISSA_BITS);+}++template <uint COMPONENT>+static METAL_FUNC half2 w4_pair_at(uint word) {+ constexpr uint offset = w4_pair_offset(COMPONENT);+ constexpr uint position = w4_pair_position(COMPONENT);+ constexpr uint mask = ((1u << W4_BITS) - 1) << position;+ constexpr uint exponent = (W4_PAIR_EXPONENT_BIAS + W4_PAIR_MANTISSA_BITS - position) << W4_PAIR_MANTISSA_BITS;+ return as_type<half2>(+ ((word >> (offset - position)) & (mask | (mask << W4_PAIR_HALF_BITS))) |+ (exponent | (exponent << W4_PAIR_HALF_BITS))+ );+}++static METAL_FUNC constexpr float w4_pair_bias_value(uint component) {+ return float(1u << (W4_PAIR_MANTISSA_BITS - w4_pair_position(component)));+}++static METAL_FUNC half2 w4_pair(uint word, ushort component) {+ switch (component) {+ case 0:+ return w4_pair_at<0>(word);+ case 1:+ return w4_pair_at<1>(word);+ case 2:+ return w4_pair_at<2>(word);+ default:+ return w4_pair_at<3>(word);+ }+}++template <uint ORIGIN>+METAL_FUNC float magic_code_float(uint code) {+ return as_type<float>(code + W8_MAGIC_FLOAT_OFFSET - ORIGIN);+}++METAL_FUNC float magic_dequant_bias(float scale) { return -12582912.0f * scale; }template <typename U, int N, int bits>inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local, const bool signed_codes) {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.hindex 77161f5ca..89528b0de 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h@@ -18,6 +18,8 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 64: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 8+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 8: 0;}@@ -34,6 +36,8 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 32: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 32+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 32: 0;}@@ -50,6 +54,8 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 256: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 256: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 256+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 64+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 128: 0;}@@ -66,6 +72,8 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 4: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 1+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 1: 0;}@@ -82,11 +90,16 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 1: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 2+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 2: 0;}constexpr uint gemm_tiling_simdgroups_per_k(GemmTiling t) {- return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4 : 1;+ return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 4+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 4+ : 1;}} // namespace gemmdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.hindex 681597107..2c4106a84 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h@@ -3,6 +3,8 @@#include <metal_stdlib>#include "../../../generated/gemm.h"+#include "../../../generated/matmul.h"+#include "../../common/quant_unpack.h"using namespace metal;@@ -13,6 +15,7 @@ namespace operands {template <GemmAPrologueKind PROLOGUE, typename Element, ushort ACTIVATION_GROUP_SIZE, bool CODES_GROUPED_BY_NIBBLE>struct LeftOperand {UZU_CONST bool QUANTIZED = PROLOGUE == GemmAPrologueKind::Int8Symmetric;+ UZU_CONST bool PACKED_HALF = PROLOGUE == GemmAPrologueKind::PackedHalf;UZU_CONST ushort BITS = QUANTIZED ? 8 : 0;UZU_CONST ushort GROUP_SIZE = QUANTIZED ? ACTIVATION_GROUP_SIZE : 0;UZU_CONST bool GROUPED_BY_NIBBLE = CODES_GROUPED_BY_NIBBLE;@@ -73,6 +76,7 @@ struct LeftStorage {const device int8_t* codes;const device float* scales;const device int32_t* group_sums;+ const device uint* packed_half;METAL_FUNC const device int32_t* correction_sums() const thread {static_assert(Left::QUANTIZED, "correction_sums is only valid for int8 activations");@@ -90,6 +94,19 @@ struct RightStorage {bool signed_codes;GemmBLayout gemm_b_layout;+ METAL_FUNC uint sign_mask() const thread {+ return signed_codes ? (Right::BITS == 4 ? W4_SIGN_MASK : W8_SIGN_MASK) : 0u;+ }++ template <typename Index>+ METAL_FUNC const device uint8_t* row_codes(+ const Index row,+ const constant uzu::matmul::GemmParams* params+ ) const thread {+ const Index row_stride = Index(params->K) / (8 / Right::BITS);+ return codes + row * row_stride;+ }+METAL_FUNC const device typename Right::ScaleElement* bias() const thread {static_assert(Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant, "bias is only valid for ScaleBiasDequant");return biases;@@ -102,6 +119,31 @@ struct RightStorage {);return zero_points;}++ template <typename Index>+ METAL_FUNC Index+ scale_index(const Index row, const Index group, const constant uzu::matmul::GemmParams* params) const thread {+ return row * Index(params->scale_output_stride) + group * Index(params->scale_group_stride);+ }++ METAL_FUNC typename Right::ScaleElement scale(+ const uint row,+ const uint group,+ const constant uzu::matmul::GemmParams* params+ ) const thread {+ return scales[scale_index(row, group, params)];+ }++ METAL_FUNC uint+ zero_point_index(const uint row, const uint group, const constant uzu::matmul::GemmParams* params) const thread {+ return row * params->zero_point_output_stride + group * params->zero_point_group_stride;+ }++ METAL_FUNC uint+ zero_point(const uint row, const uint group, const constant uzu::matmul::GemmParams* params) const thread {+ const uint index = zero_point_index(row, group, params);+ return decode_zero_point<Right::BITS>(zp() + index / ZERO_POINTS_PER_BYTE, index % ZERO_POINTS_PER_BYTE);+ }};template <typename Left, typename Element>@@ -109,12 +151,15 @@ METAL_FUNC LeftStorage<Left> pack_left(const device Element* a,const device int8_t* codes,const device float* scales,- const device int32_t* group_sums+ const device int32_t* group_sums,+ const device uint* packed_half) {- if constexpr (Left::QUANTIZED) {- return {nullptr, codes, scales, group_sums};+ if constexpr (Left::PACKED_HALF) {+ return {nullptr, nullptr, nullptr, nullptr, packed_half};+ } else if constexpr (Left::QUANTIZED) {+ return {nullptr, codes, scales, group_sums, nullptr};} else {- return {a, nullptr, nullptr, nullptr};+ return {a, nullptr, nullptr, nullptr, nullptr};}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/direct.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/direct.hnew file mode 100644index 000000000..555ee27a5--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/direct.h@@ -0,0 +1,166 @@+#pragma once++#include <metal_stdlib>++#include "../../../../../common/conversion.h"+#include "../../../../../generated/gemm.h"+#include "../../../../common/fragment.h"+#include "../../../../common/quant_unpack.h"+#include "../../../../common/simdgroup_fragment_ops.h"+#include "../../operands.h"++using namespace metal;++namespace uzu {+namespace gemm {+namespace schedules {++struct DirectSchedule {+ UZU_CONST bool TRANSPOSED_OUTPUT = false;++ template <typename Core>+ static METAL_FUNC typename Core::AccumFragment launch(+ const typename Core::LeftStorage left,+ const typename Core::RightStorage right,+ const constant uzu::matmul::GemmParams* params,+ const thread typename Core::TileContext& tile_context+ ) {+ using RightOperand = typename Core::Right;+ constexpr ushort K_STEPS_PER_BLOCK = 16;+ using Ops = typename Core::FragmentOps;+ constexpr uint K_BLOCK = Core::THREADGROUP_BLOCK_K;+ constexpr uint ACTIVATION_VECTORS_PER_LANE = K_STEPS_PER_BLOCK / 4;++ const uint lane_row = uint(tile_context.lane_position.y);+ const uint lane_column = uint(tile_context.lane_position.x);+ const uint inner_dimension = params->K;+ const uint last_weight_row = params->N - 1;+ const uint weight_sign_mask = right.sign_mask();++ const uint simdgroup_column_base = tile_context.simdgroup_column_base;+ uint weight_rows[Core::TILES_N][2];+ METAL_PRAGMA_UNROLL+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {+ METAL_PRAGMA_UNROLL+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {+ weight_rows[tile_n][row_pair] =+ min(simdgroup_column_base + Ops::FRAGMENT_COLS * tile_n + lane_column + row_pair, last_weight_row);+ }+ }++ typename Core::AccumFragment accumulator;+ accumulator.clear();++ const device bfloat* activation_row =+ left.values + min(tile_context.block_geometry.block_row_start + lane_row, params->M - 1) * inner_dimension ++ K_STEPS_PER_BLOCK * lane_column;+ const uint k_block_end = tile_context.k_block_end;++ uint4 activation_words[ACTIVATION_VECTORS_PER_LANE];+ uint4 weight_words[Core::TILES_N][2];+ float weight_scales[Core::TILES_N][2];+ float dequant_offsets[Core::TILES_N][2];++ auto load_activations = [&](uint k_block, thread uint4(&activation_dst)[ACTIVATION_VECTORS_PER_LANE]) {+ const device uint4* source = reinterpret_cast<const device uint4*>(activation_row + k_block * K_BLOCK);+ METAL_PRAGMA_UNROLL+ for (uint vector = 0; vector < ACTIVATION_VECTORS_PER_LANE; ++vector) {+ activation_dst[vector] = source[vector];+ }+ };+ auto load_weights = [&](uint k_block, thread uint4(&weight_dst)[Core::TILES_N][2]) {+ const uint weight_k_lane = k_block * K_BLOCK + K_STEPS_PER_BLOCK * lane_row;+ METAL_PRAGMA_UNROLL+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {+ METAL_PRAGMA_UNROLL+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {+ weight_dst[tile_n][row_pair] = *reinterpret_cast<const device uint4*>(+ right.row_codes(weight_rows[tile_n][row_pair], params) + weight_k_lane+ );+ }+ }+ };+ auto load_metadata =+ [&](uint k_block, thread float (&scale_dst)[Core::TILES_N][2], thread float (&offset_dst)[Core::TILES_N][2]) {+ const uint weight_group_index = (k_block * K_BLOCK + K_STEPS_PER_BLOCK * lane_row) / RightOperand::GROUP_SIZE;+ METAL_PRAGMA_UNROLL+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {+ METAL_PRAGMA_UNROLL+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {+ const uint row = weight_rows[tile_n][row_pair];+ scale_dst[tile_n][row_pair] = float(right.scale(row, weight_group_index, params));+ offset_dst[tile_n][row_pair] = magic_dequant_bias(scale_dst[tile_n][row_pair]);+ }+ }+ };++ uint k_block = tile_context.k_block_begin + tile_context.simdgroup_k_slice;+ if (k_block < k_block_end) {+ load_activations(k_block, activation_words);+ load_weights(k_block, weight_words);+ load_metadata(k_block, weight_scales, dequant_offsets);+ }+ for (; k_block < k_block_end; k_block += Core::SIMDGROUPS_PER_K) {+ const uint next_k_block = k_block + Core::SIMDGROUPS_PER_K;+ const uint prefetch_k_block = next_k_block < k_block_end ? next_k_block : k_block;+ METAL_PRAGMA_UNROLL+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {+ METAL_PRAGMA_UNROLL+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {+ weight_words[tile_n][row_pair] ^= uint4(weight_sign_mask);+ }+ }++ auto activation_generator = [&](ushort, ushort k_index) -> float2 {+ const uint word0 = reinterpret_cast<const thread uint*>(activation_words)[k_index / 2];+ const uint word1 = reinterpret_cast<const thread uint*>(activation_words)[K_STEPS_PER_BLOCK / 2 + k_index / 2];+ const float2 values0 = uzu::unpack_bfloat(word0);+ const float2 values1 = uzu::unpack_bfloat(word1);+ return k_index % 2 ? float2(values0.y, values1.y) : float2(values0.x, values1.x);+ };+ auto weight_generator = [&](ushort k_index, ushort tile_n) -> float2 {+ const uint word_index = k_index * RightOperand::BITS / 32;+ auto decode = [&](ushort row_pair) {+ return fma(+ magic_code_float<symmetric_zero_point<8>()>(+ extract_bits(weight_words[tile_n][row_pair][word_index], 8 * (k_index % 4), 8)+ ),+ weight_scales[tile_n][row_pair],+ dequant_offsets[tile_n][row_pair]+ );+ };+ return float2(decode(0), decode(1));+ };+ using ActivationFragment =+ uzu::matmul::GeneratedFragment<float, Core::TILES_M, K_STEPS_PER_BLOCK, Ops, decltype(activation_generator)>;+ using WeightFragment =+ uzu::matmul::GeneratedFragment<float, K_STEPS_PER_BLOCK, Core::TILES_N, Ops, decltype(weight_generator)>;+ ActivationFragment activation_fragment{activation_generator};+ WeightFragment weight_fragment{weight_generator};+ uzu::matmul::fragment_mma(accumulator, activation_fragment, weight_fragment);++ uint4 next_weight_words[Core::TILES_N][2];+ float next_weight_scales[Core::TILES_N][2];+ float next_dequant_offsets[Core::TILES_N][2];+ // Keep next operands separate so their loads can overlap the current MMA.+ load_activations(prefetch_k_block, activation_words);+ load_metadata(prefetch_k_block, next_weight_scales, next_dequant_offsets);+ load_weights(prefetch_k_block, next_weight_words);+ METAL_PRAGMA_UNROLL+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {+ METAL_PRAGMA_UNROLL+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {+ weight_scales[tile_n][row_pair] = next_weight_scales[tile_n][row_pair];+ dequant_offsets[tile_n][row_pair] = next_dequant_offsets[tile_n][row_pair];+ weight_words[tile_n][row_pair] = next_weight_words[tile_n][row_pair];+ }+ }+ }++ return accumulator;+ }+};++} // namespace schedules+} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.hnew file mode 100644index 000000000..3f6df944c--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h@@ -0,0 +1,131 @@+#pragma once++#include <metal_stdlib>++#include "../../../../../generated/gemm.h"+#include "../../../../common/fragment.h"+#include "../../../../common/quant_unpack.h"+#include "../../../../common/simdgroup_fragment_ops.h"+#include "../../../../common/packed_half_layout.h"++using namespace metal;++namespace uzu {+namespace gemm {+namespace schedules {++struct PackedSchedule {+ UZU_CONST bool TRANSPOSED_OUTPUT = true;++ template <ushort WORDS_PER_LANE>+ struct WeightDecoder {+ thread const uint* packed_words;++ METAL_FUNC half2 operator()(const ushort row, const ushort k) const thread {+ return w4_pair(packed_words[row * WORDS_PER_LANE + k / 4], k % 4);+ }+ };++ template <typename Core>+ static METAL_FUNC typename Core::AccumFragment launch(+ const typename Core::LeftStorage left,+ const typename Core::RightStorage right,+ const constant uzu::matmul::GemmParams* params,+ const thread typename Core::TileContext& tile_context+ ) {+ using RightOperand = typename Core::Right;+ constexpr bool HAS_BIAS = RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant;+ constexpr ushort WEIGHT_GROUP_SIZE = RightOperand::GROUP_SIZE;+ constexpr ushort K_STEPS_PER_GROUP = PackedHalfLayout<WEIGHT_GROUP_SIZE>::STEPS;+ constexpr ushort WORDS_PER_LANE = WEIGHT_GROUP_SIZE / 32;+ constexpr ushort WORDS_PER_GROUP = PackedHalfLayout<WEIGHT_GROUP_SIZE>::GROUP_WORDS;+ constexpr ushort GROUPS_PER_K_BLOCK = Core::THREADGROUP_BLOCK_K / WEIGHT_GROUP_SIZE;+ using Ops = typename Core::FragmentOps;+ using WeightFragment =+ uzu::matmul::GeneratedFragment<half, Core::TILES_M, K_STEPS_PER_GROUP, Ops, WeightDecoder<WORDS_PER_LANE>>;+ using ActivationFragment = uzu::matmul::Fragment<half, K_STEPS_PER_GROUP, Core::TILES_N, Ops>;++ const device uint* packed_activations = left.packed_half;+ const uint fragment_column_pair = uint(tile_context.lane_position.x) / 2;+ const uint lane_row = uint(tile_context.lane_position.y);+ const uint weight_sign_mask = right.sign_mask();++ typename Core::AccumFragment accumulator;+ accumulator.clear();+ const uint k_block_end = tile_context.k_block_end;+ for (uint k_block = tile_context.k_block_begin + tile_context.simdgroup_k_slice; k_block < k_block_end;+ k_block += Core::SIMDGROUPS_PER_K) {+ const uint first_weight_group = k_block * GROUPS_PER_K_BLOCK;+ for (ushort group_in_block = 0; group_in_block < GROUPS_PER_K_BLOCK; ++group_in_block) {+ const uint weight_group_index = first_weight_group + group_in_block;+ const device uint* packed_group = packed_activations + weight_group_index * WORDS_PER_GROUP;+ const device uint* activation_pairs =+ packed_group + PackedHalfLayout<WEIGHT_GROUP_SIZE>::pair_offset(fragment_column_pair, lane_row);+ ActivationFragment activation_fragment;+ METAL_PRAGMA_UNROLL+ for (ushort vector = 0; vector < K_STEPS_PER_GROUP / 4; ++vector) {+ const uint4 raw = reinterpret_cast<const device uint4*>(activation_pairs)[vector];+ activation_fragment.fragment_at(4 * vector, 0) = as_type<half2>(raw.x);+ activation_fragment.fragment_at(4 * vector + 1, 0) = as_type<half2>(raw.y);+ activation_fragment.fragment_at(4 * vector + 2, 0) = as_type<half2>(raw.z);+ activation_fragment.fragment_at(4 * vector + 3, 0) = as_type<half2>(raw.w);+ }+ const float4 activation_sums = reinterpret_cast<const device float4*>(+ packed_group + PackedHalfLayout<WEIGHT_GROUP_SIZE>::sums_offset()+ )[fragment_column_pair];++ uint weight_rows[Core::TILES_M];+ float weight_scales[Core::TILES_M];+ float weight_offsets[Core::TILES_M]; // zero point, or bias+ uint packed_weight_words[Core::TILES_M * WORDS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (ushort tile_m = 0; tile_m < Core::TILES_M; ++tile_m) {+ weight_rows[tile_m] =+ min(tile_context.simdgroup_column_base + Ops::FRAGMENT_ROWS * tile_m + lane_row, params->N - 1);+ const device uint* weight_source = reinterpret_cast<const device uint*>(+ right.row_codes(weight_rows[tile_m], params) ++ PackedHalfLayout<WEIGHT_GROUP_SIZE>::codes_offset(weight_group_index, fragment_column_pair)+ );+ METAL_PRAGMA_UNROLL+ for (ushort word_index = 0; word_index < WORDS_PER_LANE; ++word_index) {+ packed_weight_words[tile_m * WORDS_PER_LANE + word_index] = weight_source[word_index] ^ weight_sign_mask;+ }+ weight_scales[tile_m] = float(right.scale(weight_rows[tile_m], weight_group_index, params));+ if constexpr (HAS_BIAS) {+ weight_offsets[tile_m] =+ float(right.bias()[right.scale_index(weight_rows[tile_m], weight_group_index, params)]);+ } else {+ weight_offsets[tile_m] = float(right.zero_point(weight_rows[tile_m], weight_group_index, params));+ }+ }++ WeightDecoder<WORDS_PER_LANE> weight_decoder{packed_weight_words};+ WeightFragment weight_fragment{weight_decoder};+ typename Core::AccumFragment group_product;+ group_product.clear();+ uzu::matmul::fragment_mma(group_product, weight_fragment, activation_fragment);+ accumulator.for_each_lane(+ [&](ushort tile_m, ushort input_index, thread float& accumulator_value, thread float& group_product_value) {+ if constexpr (HAS_BIAS) {+ accumulator_value =+ fma(group_product_value - activation_sums[2 * input_index + 1],+ weight_scales[tile_m],+ fma(weight_offsets[tile_m], activation_sums[2 * input_index], accumulator_value));+ } else {+ const float correction =+ fma(weight_offsets[tile_m], activation_sums[2 * input_index], activation_sums[2 * input_index + 1]);+ accumulator_value = fma(group_product_value - correction, weight_scales[tile_m], accumulator_value);+ }+ },+ group_product+ );+ }+ }++ return accumulator;+ }+};++} // namespace schedules+} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.hnew file mode 100644index 000000000..eeb6f272b--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h@@ -0,0 +1,182 @@+#pragma once++#include <metal_stdlib>++#include "../../../common/thread_context.h"+#include "../../../generated/matmul.h"+#include "../generated/gemm.h"+#include "../../../common/soft_cap.h"+#include "../../common/fragment.h"+#include "../../common/simdgroup_fragment_ops.h"+#include "block_geometry.h"+#include "gemm_rht.h"+#include "gemm_tiling.h"+#include "operands.h"+#include "schedules/small_batch/packed.h"+#include "schedules/small_batch/direct.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <typename OutputElementType_, GemmTiling GEMM_TILING, typename LeftOperand, typename RightOperand>+struct SmallBatchMmaCore {+ using OutputElementType = OutputElementType_;+ using Left = LeftOperand;+ using Right = RightOperand;+ using RightElementType = typename Right::ElementType;+ using LeftStorage = operands::LeftStorage<Left>;+ using RightStorage = operands::RightStorage<Right>;+ using Schedule = metal::conditional_t<Left::PACKED_HALF, schedules::PackedSchedule, schedules::DirectSchedule>;+ using FragmentOps = uzu::matmul::SimdgroupFragmentOps;+ UZU_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);+ UZU_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);+ UZU_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);+ UZU_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);+ UZU_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);+ UZU_CONST int SIMDGROUPS_PER_K = gemm_tiling_simdgroups_per_k(GEMM_TILING);+ UZU_CONST int SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;+ UZU_CONST int SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;+ UZU_CONST ushort TILES_M = Schedule::TRANSPOSED_OUTPUT ? SIMDGROUP_BLOCK_N / FragmentOps::FRAGMENT_ROWS+ : SIMDGROUP_BLOCK_M / FragmentOps::FRAGMENT_ROWS;+ UZU_CONST ushort TILES_N = Schedule::TRANSPOSED_OUTPUT ? SIMDGROUP_BLOCK_M / FragmentOps::FRAGMENT_COLS+ : SIMDGROUP_BLOCK_N / FragmentOps::FRAGMENT_COLS;+ using AccumFragment = uzu::matmul::Fragment<float, TILES_M, TILES_N, FragmentOps>;++ using BlockGeometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>;+ struct TileContext {+ BlockGeometry block_geometry;+ ushort simd_lane_id;+ short2 lane_position;+ uint k_partition;+ ushort simdgroup_k_slice;+ uint simdgroup_column_base;+ uint k_block_begin;+ uint k_block_end;+ };++ static METAL_FUNC TileContext+ make_tile_context(const constant uzu::matmul::GemmParams* params, const thread ThreadContext& thread_context) {+ const ushort simd_lane_id = ushort(thread_context.simd_lane_id);+ const uint2 id =+ tile_id(uint2(thread_context.threadgroup_position.x, thread_context.threadgroup_position.y), params);+ const BlockGeometry block_geometry = BlockGeometry::compute(id, params);+ const ushort simdgroup_index = ushort(thread_context.simdgroup_index);+ const uint total_blocks = params->K / THREADGROUP_BLOCK_K;+ const uint blocks_per_partition =+ SIMDGROUPS_PER_K * params->aligned_inner_iterations + total_blocks % SIMDGROUPS_PER_K;+ const uint k_block_begin = thread_context.threadgroup_position.z * blocks_per_partition;+ return {+ block_geometry,+ simd_lane_id,+ FragmentOps::get_position(simd_lane_id),+ thread_context.threadgroup_position.z,+ ushort(simdgroup_index / SIMDGROUPS_PER_COLUMN),+ block_geometry.block_col_start + uint(simdgroup_index % SIMDGROUPS_PER_COLUMN) * SIMDGROUP_BLOCK_N,+ k_block_begin,+ k_block_begin + blocks_per_partition+ };+ }++ static METAL_FUNC void run(+ LeftStorage left,+ RightStorage right,+ device OutputElementType* d,+ const constant uzu::matmul::GemmParams* params,+ GemmDTransform output_transform,+ const constant float& soft_cap,+ const device RightElementType* output_bias,+ const device int32_t* rht_factors,+ threadgroup RightElementType* scratch,+ const thread ThreadContext& thread_context+ ) {+ TileContext tile_context = make_tile_context(params, thread_context);+ if (tile_context.block_geometry.out_of_bounds) {+ return;+ }+ AccumFragment accumulator = Schedule::template launch<SmallBatchMmaCore>(left, right, params, tile_context);+ const bool output_writer = accumulator.template reduce_k_slices<SIMDGROUPS_PER_K, SIMDGROUPS_PER_COLUMN>(+ reinterpret_cast<threadgroup float*>(scratch),+ thread_context+ );+ const bool needs_rht = output_transform.contains(GemmDTransform::RHT);+ if (!output_writer && !needs_rht) {+ return;+ }++ const uint row_start = tile_context.block_geometry.block_row_start;+ const uint m_remaining = min(uint(THREADGROUP_BLOCK_M), params->M - row_start);+ const uint n_remaining = tile_context.simdgroup_column_base < params->N+ ? min(uint(SIMDGROUP_BLOCK_N), params->N - tile_context.simdgroup_column_base)+ : 0;+ const uint safe_output_col = min(tile_context.simdgroup_column_base, params->N - 1);+ const short2 tile_dimensions = Schedule::TRANSPOSED_OUTPUT ? short2(short(m_remaining), short(n_remaining))+ : short2(short(n_remaining), short(m_remaining));+ device OutputElementType* row_destination =+ d + size_t(tile_context.k_partition) * size_t(params->M) * size_t(params->N) ++ size_t(row_start) * params->leading_dimension_d;+ if (output_writer) {+ if (output_transform.contains(GemmDTransform::SCALE)) {+ const uzu::matmul::TransformScale<float, float> scale(params->ab_scale);+ accumulator.map([&](float value) { return scale.apply(value); });+ }+ const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);+ if (needs_bias && !needs_rht) {+ accumulator.map_coords(tile_context.simd_lane_id, [&](short row, short col, float value) {+ const uint bias_col = Schedule::TRANSPOSED_OUTPUT ? uint(row) : uint(col);+ return bias_col < n_remaining ? value + float(output_bias[tile_context.simdgroup_column_base + bias_col])+ : value;+ });+ }+ if (output_transform.contains(GemmDTransform::SOFT_CAP)) {+ accumulator.map([&](float value) { return uzu::apply_soft_cap(value, soft_cap); });+ }++ device OutputElementType* destination = row_destination + safe_output_col;+ const int row_stride = Schedule::TRANSPOSED_OUTPUT ? 1 : int(params->leading_dimension_d);+ const auto col_stride = [&]() {+ if constexpr (Schedule::TRANSPOSED_OUTPUT) {+ return int(params->leading_dimension_d);+ } else {+ return uzu::Int<1>{};+ }+ }();+ accumulator.store_safe(+ tile_context.simd_lane_id,+ destination,+ row_stride,+ tile_dimensions,+ uzu::Int<1>{},+ uzu::Int<1>{},+ col_stride+ );+ }++ if (needs_rht) {+ const uint tile_col_start = tile_context.block_geometry.block_col_start;+ const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(tile_col_start)));+ device OutputElementType* tile_output = row_destination + tile_col_start;+ const device RightElementType* tile_bias = output_bias;+ if (output_transform.contains(GemmDTransform::BIAS)) {+ tile_bias += tile_col_start;+ }++ threadgroup_barrier(mem_flags::mem_device);+ apply_output_random_hadamard_transform(+ tile_output,+ rht_factors + tile_col_start,+ tile_bias,+ output_transform.contains(GemmDTransform::BIAS),+ ushort(m_remaining),+ tile_block_cols,+ params->leading_dimension_d,+ SIMDGROUPS_PER_K * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN,+ thread_context+ );+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 19f87e0bd..5c7f457ff 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -6,15 +6,21 @@#include "common/gemm_tiling.h"#include "common/mxu_mma_core.h"#include "common/simdgroup_mma_core.h"+#include "common/small_batch_mma_core.h"using namespace metal;using namespace uzu::gemm;#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)+#define GEMM_PACKED_HALF_TILING (GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4)+#define GEMM_SMALL_BATCH (GEMM_PACKED_HALF_TILING || GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4)+#define GEMM_SMALL_BATCH_W8 (GEMM_SMALL_BATCH && !GEMM_PACKED_HALF_TILING)#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant)#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)#define GEMM_TGA_ELEMENTS \- ((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))+ ((GEMM_SMALL_BATCH || USE_MXU) \+ ? 1 \+ : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))#define GEMM_K_PARTIAL_ELEMENTS \(gemm_tiling_simdgroups_per_k(GEMM_TILING) == 1 \? 1 \@@ -25,7 +31,9 @@ using namespace uzu::gemm;#define GEMM_TGB_ELEMENTS \((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \: GEMM_K_PARTIAL_ELEMENTS) \- : (gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))+ : (GEMM_SMALL_BATCH \+ ? GEMM_K_PARTIAL_ELEMENTS \+ : gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))template <typename AT,@@ -55,7 +63,9 @@ VARIANTS(GemmTiling::Tile32x64x256_Simdgroups2x2,GemmTiling::Tile64x32x256_Simdgroups4x1,GemmTiling::Tile64x64x256_Simdgroups2x2,- GemmTiling::Tile128x128x256_Simdgroups4x4)+ GemmTiling::Tile128x128x256_Simdgroups4x4,+ GemmTiling::Tile8x32x64_Simdgroups1x2x4,+ GemmTiling::Tile8x32x128_Simdgroups1x2x4)VARIANTS(TRANSPOSE_B, false, true)VARIANTS(USE_MXU, false, true)VARIANTS(@@ -69,7 +79,8 @@ VARIANTS(GROUP_SIZE, 0, 32, 64)VARIANTS(A_PROLOGUE,GemmAPrologueKind::FullPrecision,- GemmAPrologueKind::Int8Symmetric)+ GemmAPrologueKind::Int8Symmetric,+ GemmAPrologueKind::PackedHalf)VARIANTS(A_GROUP_SIZE, 0, 128)CONSTRAINT(USE_MXU ==@@ -105,13 +116,21 @@ CONSTRAINT(CONSTRAINT(GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||(TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)+CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || A_PROLOGUE == GemmAPrologueKind::PackedHalf || USE_MXU)+CONSTRAINT(GEMM_PACKED_HALF_TILING == (A_PROLOGUE == GemmAPrologueKind::PackedHalf))+CONSTRAINT(!GEMM_PACKED_HALF_TILING || (!USE_MXU && TRANSPOSE_B && BITS == 4))+CONSTRAINT(+ !GEMM_SMALL_BATCH_W8 ||+ (!USE_MXU && TRANSPOSE_B && A_PROLOGUE == GemmAPrologueKind::FullPrecision &&+ BITS == 8))CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision ||(TRANSPOSE_B && B_PROLOGUE != GemmBPrologueKind::FullPrecision))-CONSTRAINT((A_PROLOGUE == GemmAPrologueKind::FullPrecision) == (A_GROUP_SIZE == 0))+CONSTRAINT((A_PROLOGUE != GemmAPrologueKind::Int8Symmetric) == (A_GROUP_SIZE == 0))+CONSTRAINT(!A_IS_INT8 || GEMM_TILING != GemmTiling::Tile128x128x256_Simdgroups4x4)KERNEL(Gemm)(const device AT* a OPTIONAL(A_PROLOGUE == GemmAPrologueKind::FullPrecision),+ const device uint* a_packed_half OPTIONAL(A_PROLOGUE == GemmAPrologueKind::PackedHalf),const device BT* b,device DT* d,const device BT* scales@@ -124,6 +143,8 @@ KERNEL(Gemm)(OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),const device int32_t* rht_factorsOPTIONAL(output_transform.contains(GemmDTransform::RHT)),+ const constant float& soft_cap+ OPTIONAL(output_transform.contains(GemmDTransform::SOFT_CAP)),const device int8_t* a_int8 OPTIONAL(A_IS_INT8),const device float* a_scales OPTIONAL(A_IS_INT8),const device int32_t* a_group_sums OPTIONAL(NEEDS_ASYMMETRIC_WEIGHT_CORRECTION),@@ -159,7 +180,7 @@ KERNEL(Gemm)(NEEDS_ASYMMETRIC_WEIGHT_CORRECTION == (A_IS_INT8 && RightOperand::NEEDS_CORRECTION),"kernel bindings and operand correction policy must agree");- const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums);+ const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums, a_packed_half);const auto right_storage =operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes, gemm_b_layout);@@ -177,6 +198,20 @@ KERNEL(Gemm)(b_shared,thread_context);+ } else if constexpr (GEMM_SMALL_BATCH) {+ using Core = SmallBatchMmaCore<DT, GEMM_TILING, LeftOperand, RightOperand>;+ Core::run(+ left_storage,+ right_storage,+ d,+ params,+ output_transform,+ soft_cap,+ output_bias,+ rht_factors,+ b_shared,+ thread_context+ );} else {using Core = SimdgroupMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;Core::run(diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 97c81313a..c1da24799 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -258,7 +258,7 @@ impl GemmKernel {let use_mxu = plan.engine == GemmEngine::Mxu;let gemm_b_layout = quantized.as_ref().map_or(GemmBLayout::RowMajor, |quantized| quantized.weights_layout);- if gemm_b_layout == GemmBLayout::MxuTiled && (!use_mxu || shape.a_full_precision) {+ if gemm_b_layout == GemmBLayout::MxuTiled && (!use_mxu || !shape.a_is_quantized()) {return Err(MatmulMetalError::IncompatibleA {path: "Gemm",reason: "MxuTiled GEMM B layout requires int8 activations on the MXU",@@ -282,23 +282,26 @@ impl GemmKernel {quantized.zero_point_strides(),),};- let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match &a {+ let (a_full_precision, a_packed_half, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match &a {MatmulA::FullPrecision {values,offset,- } => (Some(values.subrange(*offset..)), None, None, None, GemmAPrologueKind::FullPrecision, None),+ } => (Some(values.subrange(*offset..)), None, None, None, None, GemmAPrologueKind::FullPrecision, None),+ MatmulA::PackedHalf {+ values,+ ..+ } => (None, Some(values.subrange(0..)), None, None, None, GemmAPrologueKind::PackedHalf, None),MatmulA::Int8Symmetric {values,scales: activation_scales,group_sums: activation_group_sums,- scale_group_size,- code_layout,+ quantization,} => {validate_int8_left_operand(use_mxu,shape,- *scale_group_size,- *code_layout,+ quantization.scale_group_size(),+ quantization.code_layout(),activation_group_sums.is_some(),)?;if output_transform.contains(GemmDTransform::SOFT_CAP) {@@ -308,12 +311,13 @@ impl GemmKernel {});}(+ None,None,Some(*values),Some(*activation_scales),*activation_group_sums,GemmAPrologueKind::Int8Symmetric,- Some(*scale_group_size),+ Some(quantization.scale_group_size()),)},MatmulA::Trellis {@@ -398,6 +402,7 @@ impl GemmKernel {let kernel = self.get_or_create(command_buffer.context(), specialization, self.output_data_type)?;kernel.encode(a_full_precision,+ a_packed_half,weights,d.reborrow(),scales,@@ -405,6 +410,7 @@ impl GemmKernel {zero_points,output_bias,rht_factors,+ soft_cap,a_int8,a_scales,a_group_sums,@@ -438,20 +444,29 @@ impl GemmKernel {command_buffer: &mut MetalCommandBufferEncoding,) -> Result<(), MatmulMetalError> {let (m, n, k) = (shape.m, shape.n, shape.k);- let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {+ let (a_full_precision, a_packed_half, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {MatmulA::FullPrecision {values,offset,- } => (Some(values.subrange(offset..)), None, None, None, GemmAPrologueKind::FullPrecision, None),+ } => (Some(values.subrange(offset..)), None, None, None, None, GemmAPrologueKind::FullPrecision, None),+ MatmulA::PackedHalf {+ values,+ ..+ } => (None, Some(values.subrange(0..)), None, None, None, GemmAPrologueKind::PackedHalf, None),MatmulA::Int8Symmetric {values,scales,group_sums,- scale_group_size,- code_layout: _,- } => {- (None, Some(values), Some(scales), group_sums, GemmAPrologueKind::Int8Symmetric, Some(scale_group_size))- },+ quantization,+ } => (+ None,+ None,+ Some(values),+ Some(scales),+ group_sums,+ GemmAPrologueKind::Int8Symmetric,+ Some(quantization.scale_group_size()),+ ),MatmulA::Trellis {..} => {@@ -481,7 +496,7 @@ impl GemmKernel {)?;let elem = (m as usize) * (n as usize);- let partial_data_type = if a_int8.is_some() {+ let partial_data_type = if a_int8.is_some() || tiling.is_small_batch_variant() {DataType::F32} else {self.output_data_type@@ -493,6 +508,7 @@ impl GemmKernel {let part_kernel = self.get_or_create(command_buffer.context(), part_spec, partial_data_type)?;part_kernel.encode(a_full_precision,+ a_packed_half,weights,&mut temp,scales,@@ -500,6 +516,7 @@ impl GemmKernel {zero_points,None::<&<Metal as Backend>::GlobalGpuBuffer>,None::<&<Metal as Backend>::GlobalGpuBuffer>,+ None,a_int8,a_scales,a_group_sums,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rsindex 5ce0acd5e..7f4107a20 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -8,7 +8,7 @@ mod trellis;pub use error::GemmSpecializationError;pub use kernel::GemmKernel;pub use selection::GemmPlanError;-pub(super) use selection::GemmProblem;+pub(super) use selection::{GemmProblem, small_batch_format, small_m_weights};use crate::backends::metal::gpu_types::gemm::GemmTiling;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 1c4894164..0c335dc3e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -4,11 +4,17 @@ use thiserror::Error;use crate::{backends::{common::{- gpu_types::gemm::GemmBPrologueKind,- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},+ gpu_types::{+ gemm::{GemmBPrologueKind, GemmDTransform},+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ },+ kernel::{+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,+ matmul::{ActivationFormat, MatmulShape},+ },},metal::{- gpu_types::gemm::GemmTiling,+ gpu_types::gemm::{GemmBLayout, GemmTiling},kernel::matmul::gemm::{GemmEngine, GemmPlan, policy},},},@@ -34,6 +40,40 @@ pub enum GemmPlanError {UnsupportedLayout(&'static str),}+/// The weight formats with small-M kernels (batched GEMV and register GEMM): W4 zero-point or scale-bias, and W8+/// symmetric, at group size 32 or 64. Returns the GEMM's activation format, tiling and K alignment.+pub fn small_m_weights(shape: &MatmulShape) -> Option<(ActivationFormat, GemmTiling, u32)> {+ let group_size = shape.b_group_size.filter(|group_size| matches!(group_size, 32 | 64))?;+ match (shape.b_prologue, shape.b_bits) {+ (GemmBPrologueKind::ScaleZeroPointDequant | GemmBPrologueKind::ScaleBiasDequant, Some(4)) => Some((+ ActivationFormat::PackedHalf {+ group_size,+ },+ GemmTiling::Tile8x32x64_Simdgroups1x2x4,+ 512,+ )),+ (GemmBPrologueKind::ScaleSymmetricDequant, Some(8)) => {+ Some((ActivationFormat::Bf16, GemmTiling::Tile8x32x128_Simdgroups1x2x4, 256))+ },+ _ => None,+ }+}++pub fn small_batch_format(+ shape: MatmulShape,+ layout: GemmBLayout,+) -> Option<(ActivationFormat, GemmTiling)> {+ let (activation, tiling, k_alignment) = small_m_weights(&shape)?;+ (shape.b_transpose+ && !shape.gathered+ && !shape.b_is_trellis+ && layout == GemmBLayout::RowMajor+ && shape.b_leading_dimension.is_none()+ && shape.k.is_multiple_of(k_alignment)+ && (!shape.d_transform.contains(GemmDTransform::RHT) || shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)))+ .then_some((activation, tiling))+}+impl GemmProblem {pub fn new(shape: MatmulShape,@@ -63,6 +103,23 @@ impl GemmProblem {self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family))}+ pub fn small_batch_plan(+ self,+ layout: GemmBLayout,+ ) -> Option<GemmPlan> {+ let (activation, tiling) = small_batch_format(self.shape, layout)?;+ let plan = self.finish_plan(GemmEngine::Simdgroup, tiling);+ let partition_alignment = if tiling.is_packed_w4() || plan.split_k == 1 {+ 256+ } else {+ 512+ };+ (self.shape.a == activation+ && (4..=8).contains(&self.shape.m)+ && self.shape.k.is_multiple_of(partition_alignment * plan.split_k))+ .then_some(plan)+ }+#[cfg(test)]pub(super) fn select_plan_for_engine(self,@@ -123,9 +180,9 @@ impl GemmProblem {return 1;};let align = step.max(ACTIVATION_SCALE_GROUP_SIZE) * tiling.simdgroups_k();- let target_tiles = policy::split_k_target_tiles(!shape.a_full_precision, tiling, shape.b_bits);+ let target_tiles = policy::split_k_target_tiles(shape.a_is_quantized(), tiling, shape.b_bits);let mut split_k = (target_tiles / base_tiles).max(1).min((shape.k / align).max(1));- if !shape.a_full_precision && engine == GemmEngine::Mxu && tiling.block_k() != 0 {+ if shape.a_is_quantized() && engine == GemmEngine::Mxu && tiling.block_k() != 0 {split_k = split_k.min((shape.k / tiling.block_k()).max(1));}while split_k > 1 && !shape.k.is_multiple_of(split_k * align) {@@ -198,12 +255,12 @@ pub(super) fn outer_block_k(}fn mxu_is_eligible(shape: MatmulShape) -> bool {- if !shape.a_full_precision || shape.b_prologue == GemmBPrologueKind::FullPrecision {+ if shape.a_is_quantized() || shape.b_prologue == GemmBPrologueKind::FullPrecision {return true;}shape.b_transpose&& shape.b_leading_dimension.is_none()- && shape.k.is_multiple_of(policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n).block_k())+ && shape.k.is_multiple_of(policy::mxu_mn_tile(shape.a_is_quantized(), shape.m, shape.n).block_k())}fn select_tiling(@@ -214,7 +271,7 @@ fn select_tiling(match engine {GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),- GemmEngine::Mxu if !shape.a_full_precision => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {+ GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {tilingif tiling.simdgroups_k() > 1&& !shape.k.is_multiple_of(tiling.simdgroups_k() * ACTIVATION_SCALE_GROUP_SIZE) =>diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 1b1c4b7f6..6201cb429 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -61,6 +61,7 @@ impl GemmSpecialization {let valid_activation_group = match self.a_prologue {GemmAPrologueKind::FullPrecision => self.a_group_size.is_none(),GemmAPrologueKind::Int8Symmetric => self.a_group_size == Some(ACTIVATION_SCALE_GROUP_SIZE),+ GemmAPrologueKind::PackedHalf => self.a_group_size.is_none(),};if !valid_activation_group {return Err(GemmSpecializationError::InvalidAGroupSize {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex b374e8727..041cf9a00 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -12,6 +12,19 @@ using namespace metal;using namespace uzu;using namespace uzu::gemm;+#define GEMV_BATCHED_INPUT (INPUT_ALIGNED && GROUP_LANES == 1)+#define GEMV_FULL_PRECISION_TILE \+ (BITS == 0 && NUM_SIMDGROUPS == 8 && REDUCTION_LANES == 32 && GROUP_LANES == 1 && (INPUT_ALIGNED || K_SPLIT == 1) && \+ (OUTPUT_ROW_TILE == 8 / K_SPLIT || OUTPUT_ROW_TILE == 32 / K_SPLIT) && INPUT_ROW_TILE == 1)+#define GEMV_QUANTIZED_ROW_TILE \+ (K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \+ ((BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))+#define GEMV_BATCHED_FMA_32_TILE \+ (K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \+ INPUT_ROW_TILE <= 4)+#define GEMV_BATCHED_FMA_16_TILE \+ (K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 2 && OUTPUT_ROW_TILE == 16 && INPUT_ROW_TILE >= 2 && \+ INPUT_ROW_TILE <= 4)template <typename AT,typename BT,@@ -41,66 +54,22 @@ VARIANTS(GROUP_SIZE, 0, 32, 64)VARIANTS(BITS, 0, 4, 8)VARIANTS(K_SPLIT, 1, 2, 4, 8)VARIANTS(INPUT_ALIGNED, false, true)-VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4, 5, 6, 7, 8)-VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32, 64)-VARIANTS(REDUCTION_LANES, 8, 16, 32)-VARIANTS(GROUP_LANES, 1, 2, 4, 8, 16)+VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4)+VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32)+VARIANTS(REDUCTION_LANES, 8, 32)+VARIANTS(GROUP_LANES, 1, 2, 4, 8)VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)-CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))-CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)-CONSTRAINT(BITS == 0 || K_SPLIT == 1)-CONSTRAINT(BITS != 0 || (INPUT_ROW_TILE == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && GROUP_LANES == 1))-CONSTRAINT(INPUT_ROW_TILE != 1 || REDUCTION_LANES == 32)CONSTRAINT(- INPUT_ROW_TILE == 1 ||- (K_SPLIT == 1 && INPUT_ALIGNED && AT == "bfloat" && DT == "bfloat" && GROUP_LANES == 1))--#define GEMV_TILE(OUTPUT_ROWS, LANES, SIMD_GROUPS) \- (OUTPUT_ROW_TILE == OUTPUT_ROWS && REDUCTION_LANES == LANES && NUM_SIMDGROUPS == SIMD_GROUPS)-#define INPUT_ROWS(FIRST, LAST) (INPUT_ROW_TILE >= FIRST && INPUT_ROW_TILE <= LAST)--CONSTRAINT(- INPUT_ROW_TILE == 1 ||- (B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant && GROUP_SIZE == 64 &&- ((GEMV_TILE(16, 8, 2) && INPUT_ROWS(2, 7)) ||- (BITS == 4 && GEMV_TILE(16, 16, 4) && INPUT_ROWS(3, 6)) ||- (BITS == 4 && GEMV_TILE(8, 8, 2) && INPUT_ROW_TILE == 2))))--// Keep only selector-reachable geometry families.-CONSTRAINT(- BITS == 0 ||- (NUM_SIMDGROUPS == 2 &&- (OUTPUT_ROW_TILE == 2 || OUTPUT_ROW_TILE == 4 || OUTPUT_ROW_TILE == 8 ||- (INPUT_ROW_TILE > 1 && OUTPUT_ROW_TILE == 16))) ||- (NUM_SIMDGROUPS == 4 && (OUTPUT_ROW_TILE == 8 || OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)) ||- (NUM_SIMDGROUPS == 8 && (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)))-CONSTRAINT(- BITS == 0 || (AT == "bfloat" && DT == "bfloat") ||- (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32))-CONSTRAINT(BITS != 8 || (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32) || INPUT_ROW_TILE > 1)-CONSTRAINT(INPUT_ALIGNED || K_SPLIT == 1)-CONSTRAINT(K_SPLIT <= NUM_SIMDGROUPS && NUM_SIMDGROUPS % K_SPLIT == 0)-CONSTRAINT(OUTPUT_ROW_TILE % (NUM_SIMDGROUPS / K_SPLIT) == 0)-CONSTRAINT(OUTPUT_ROW_TILE / (NUM_SIMDGROUPS / K_SPLIT) % (32 / REDUCTION_LANES) == 0)-CONSTRAINT(- (BITS == 0 && (OUTPUT_ROW_TILE == (NUM_SIMDGROUPS / K_SPLIT) ||- OUTPUT_ROW_TILE == 4 * (NUM_SIMDGROUPS / K_SPLIT))) ||- (BITS != 0 &&- (OUTPUT_ROW_TILE == NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 2 * NUM_SIMDGROUPS ||- OUTPUT_ROW_TILE == 4 * NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 8 * NUM_SIMDGROUPS)))-CONSTRAINT(REDUCTION_LANES % GROUP_LANES == 0)-CONSTRAINT(BITS == 0 || GROUP_SIZE % GROUP_LANES == 0)-CONSTRAINT(BITS != 4 || (GROUP_SIZE / GROUP_LANES) % 16 == 0)-CONSTRAINT(BITS != 8 || (GROUP_SIZE / GROUP_LANES) % 8 == 0)-CONSTRAINT(BITS != 0 || REDUCTION_LANES == 32)+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision && BITS == 0 && GROUP_SIZE == 0) ||+ ((B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant ||+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) &&+ BITS == 4 && GROUP_SIZE != 0) ||+ (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0))+CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)CONSTRAINT(- BITS == 0 || INPUT_ROW_TILE > 1 ||- (BITS == 4 && ((GROUP_SIZE == 32 && GROUP_LANES == 2) || (GROUP_SIZE == 64 && GROUP_LANES == 4))) ||- (BITS == 8 && ((GROUP_SIZE == 32 && GROUP_LANES == 4) || (GROUP_SIZE == 64 && GROUP_LANES == 8))))+ GEMV_FULL_PRECISION_TILE ||+ (BITS != 0 && (GEMV_QUANTIZED_ROW_TILE || GEMV_BATCHED_FMA_32_TILE || GEMV_BATCHED_FMA_16_TILE)))KERNEL(Gemv)(const device uint32_t* b,const device BT* scalesdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 0a29f270d..7c2f56b33 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -1,5 +1,3 @@-use metal::MTLGPUFamily;-use crate::{backends::{common::{@@ -58,7 +56,6 @@ impl GemvSpecialization {input_data_type: DataType,output_data_type: DataType,gpu_core_count: u32,- apple_gpu_family: MTLGPUFamily,) -> Option<Self> {let is_quant = shape.is_quant();let bits = shape.b_bits.unwrap_or(0);@@ -74,51 +71,26 @@ impl GemvSpecialization {let tile = if is_quant && shape.gathered {policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)} else if is_quant {- policy::quantized_tile(- gpu_core_count,- apple_gpu_family,- bits,- shape.b_group_size.unwrap_or(0),- shape.m,- shape.n,- shape.k,- bf16_io,- )- .map(|tile| match gemm_b_layout {- GemmBLayout::MxuTiled if mxu_tiled_kernel => {- policy::GemvTile::quantized_output_tile(8, 32, 1, 32, tile.group_lanes)- },- GemmBLayout::MxuTiled if tile.output_row_tile() < 16 => {- policy::GemvTile::quantized_output_tile(8, 16, 1, 32, tile.group_lanes)- },- _ => tile,- })+ policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)} else {- let mixed_precision = weights_data_type == DataType::F32- && (input_data_type != DataType::F32 || output_data_type != DataType::F32);- if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {+ if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {return None;}let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);- Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))+ Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned))};- Self::select_tile(shape, gemm_b_layout, weights_data_type, input_data_type, output_data_type, tile?).map(- |specialization| Self {- mxu_tiled_kernel,- ..specialization- },- )+ Self::select_tile(shape, gemm_b_layout, tile?).map(|specialization| Self {+ mxu_tiled_kernel,+ ..specialization+ })}pub fn select_tile(shape: &MatmulShape,gemm_b_layout: GemmBLayout,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,tile: policy::GemvTile,) -> Option<Self> {- if !shape.b_transpose || !shape.a_full_precision {+ if !shape.b_transpose || shape.a != crate::backends::common::kernel::matmul::ActivationFormat::Bf16 {return None;}let is_quant = shape.is_quant();@@ -130,14 +102,9 @@ impl GemvSpecialization {if bad_leading_dimension {return None;}- let output_transform = output_transform_for_tile(shape, tile)?;let bits = shape.b_bits.unwrap_or(0);- if !is_quant {- let mixed_precision = weights_data_type == DataType::F32- && (input_data_type != DataType::F32 || output_data_type != DataType::F32);- if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {- return None;- }+ if !is_quant && (shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH) {+ return None;}let block_size = if !is_quant {FP_K_BLOCK@@ -147,10 +114,11 @@ impl GemvSpecialization {256};let input_aligned = shape.k.is_multiple_of(block_size);- // Gathered quantized rows cannot share one input tile.- if is_quant && shape.gathered && tile.input_row_tile > 1 {+ // Gathered quantized rows cannot share one input tile, and batched tiles need whole K blocks.+ if is_quant && tile.input_row_tile > 1 && (shape.gathered || !input_aligned) {return None;}+ let output_transform = output_transform_for_tile(shape, tile)?;let specialization = Self {b_prologue: shape.b_prologue,group_size: shape.b_group_size.unwrap_or(0),diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rsindex cc988e389..ce64b7eb8 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs@@ -1,7 +1,3 @@-use metal::MTLGPUFamily;--use crate::backends::metal::context::LARGE_MIN_GPU_CORES;-mod quantized;pub(super) use quantized::{gathered_tile, select as quantized_tile};@@ -28,10 +24,6 @@ pub struct GemvTile {pub(super) group_lanes: u32,}-const SMALL_G13_HUGE_N: u32 = 32768;-const SMALL_G13_BROAD_ROW_N: u32 = 6144;-const DEEP_K: u32 = 8192;-const FP_LARGE_SPLIT_K_MIN_DEPTH: u32 = 4 * FP_K_BLOCK;const FP_K_DEPTH_N_MAX: u32 = 4095;const FP_K_DEPTH_DEEP_MIN: u32 = 3072;const FP_K_DEPTH_VERY_DEEP_RATIO: u32 = 16;@@ -142,29 +134,18 @@ fn preferred_fp_k_split(/// Selects the full-precision GEMV tile. `m` is the input-vector count,/// `n` is the output row count, and `k` is the reduction depth.pub(super) fn fp_tile(- gpu_core_count: u32,- apple_gpu_family: MTLGPUFamily,m: u32,n: u32,k: u32,input_aligned: bool,) -> GemvTile {- let is_large_gpu = gpu_core_count >= LARGE_MIN_GPU_CORES;- let is_small_legacy = !is_large_gpu && apple_gpu_family < MTLGPUFamily::Apple8;- // SG8 is the portable full-precision geometry.- let should_disable_k_split = !input_aligned- || (m == 1 && is_large_gpu && k < FP_LARGE_SPLIT_K_MIN_DEPTH)- || (m == 1 && is_small_legacy && n >= SMALL_G13_HUGE_N);-- let k_split = if should_disable_k_split {- 1- } else {+ // SG8 is the portable full-precision geometry; one output row per SIMD group for a single vector.+ let k_split = if input_aligned {cap_k_split_to_complete_fp_k_blocks(k, preferred_fp_k_split(m, n, k))+ } else {+ 1};-- let results_per_simdgroup = if is_small_legacy && m == 1 && n >= SMALL_G13_BROAD_ROW_N {- DEFAULT_RESULTS_PER_SIMDGROUP- } else if m == 1 && (k <= DEEP_K || !is_large_gpu) {+ let results_per_simdgroup = if m == 1 {1} else {DEFAULT_RESULTS_PER_SIMDGROUPdiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rsindex 43dd010ff..035b8a042 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs@@ -1,10 +1,4 @@-use metal::MTLGPUFamily;-use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile};-use crate::backends::metal::context::LARGE_MIN_GPU_CORES;--const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];-const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];/// Lane-sliced tiles split each quantization group across reduction lanes./// A lane stages 8 bytes: 16 W4 values or 8 W8 values.@@ -42,86 +36,16 @@ pub fn gathered_tile((n >= tile.rows_per_lane()).then_some(tile)}-fn table_bucket_index(- value: u32,- bucket_maxes: &[u32],-) -> usize {- bucket_maxes.partition_point(|&max| value > max)-}--fn lane_policy(- gpu_core_count: u32,- apple_gpu_family: MTLGPUFamily,- m: u32,- n: u32,- k: u32,- bits: u32,- group: u32,-) -> GemvTile {- let is_large_gpu = gpu_core_count >= LARGE_MIN_GPU_CORES;- if m != 1 || bits != 4 {- return lane_default(bits, group);- }-- let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);- let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);- let (num_simdgroups, results_per_simdgroup) = match (is_large_gpu, apple_gpu_family, k_bucket, n_bucket) {- (true, _, 0, 1) => (4, 2),- (true, _, 1, 0) => (2, 1),- (true, _, 1, 1..=3) => (2, 2),- (true, _, 1, 4) => (2, 1),- (true, _, 1, 5) => (2, 4),- (true, _, 2, 1) => (4, 2),- (true, _, 3, 1) => (2, 2),- (false, MTLGPUFamily::Apple9, 0, 1) => (4, 4),- (false, MTLGPUFamily::Apple9, 1, 0) => (4, 2),- (false, MTLGPUFamily::Apple9, 1, 1) => (2, 2),- (false, MTLGPUFamily::Apple9, 1, 2) => (4, 2),- (false, MTLGPUFamily::Apple9, 1, 4) => (2, 2),- (false, MTLGPUFamily::Apple9, 1, 5) => (4, 2),- (false, MTLGPUFamily::Apple9, 2, 1) => (4, 2),- (false, MTLGPUFamily::Apple9, 3, 1) => (2, 2),- (false, MTLGPUFamily::Apple8, 0, 1) => (4, 4),- (false, MTLGPUFamily::Apple8, 1, _) | (false, MTLGPUFamily::Apple8, 2, 1) => (8, 2),- (false, family, 0, 1) if family < MTLGPUFamily::Apple8 => (4, 8),- (false, family, 1, 0..=3) if family < MTLGPUFamily::Apple8 => (8, 2),- _ => (8, 4),- };- let selected = lane_tile(num_simdgroups, results_per_simdgroup, bits, group);- if n < selected.results_per_simdgroup {- lane_default(bits, group)- } else {- selected- }-}-pub fn select(- gpu_core_count: u32,- apple_gpu_family: MTLGPUFamily,bits: u32,group: u32,m: u32,n: u32,- k: u32,bf16_io: bool,) -> Option<GemvTile> {- if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {- return None;- }-- if !bf16_io {- let tile = lane_default(bits, group);- return (m <= 4 && n >= tile.rows_per_lane()).then_some(tile);- }-- let tile = match m {- 1..=4 => lane_policy(gpu_core_count, apple_gpu_family, m, n, k, bits, group),- 5..=8 if n <= 64 => lane_default(bits, group),- _ => return None,- };- // Input rows are tiled independently; partial tiles clamp loads and stores.- if n < tile.rows_per_lane() {+ // The GEMV pipelines exist for BF16 I/O only. Rows 5..=8 stay on GEMV for tiny N only.+ if !bf16_io || (m > 4 && n > 64) {return None;}- Some(tile)+ gathered_tile(bits, group, m, n)}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex dc5becb57..0f8cb2afe 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -20,11 +20,10 @@ use crate::{gpu_types::gemm::{GemmBLayout, GemmTiling},kernel::matmul::{GemmKernel, MatmulMetalError,- gemm::{self, GemmPlan, GemmProblem},- gemv::{GemvKernel, GemvSpecialization},+ gemm::{self, GemmPlan, GemmProblem, small_batch_format, small_m_weights},+ gemv::{GemvKernel, GemvSpecialization, GemvTile},matmul_dispatch::MatmulDispatch,matmul_output_work::MatmulOutputWork,- qmv::{self, QmvRoute},},},},@@ -84,7 +83,6 @@ impl MatmulMetalKernel {pub fn choose_dispatch(shape: &MatmulShape,gemm_b_layout: GemmBLayout,- device_name: &str,gpu_core_count: u32,apple_gpu_family: MTLGPUFamily,supports_mxu: bool,@@ -92,28 +90,23 @@ impl MatmulMetalKernel {input_data_type: DataType,output_data_type: DataType,) -> MatmulDispatch {- let all_bf16 = weights_data_type == DataType::BF16- && input_data_type == DataType::BF16- && output_data_type == DataType::BF16;let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);if shape.b_is_trellis {return MatmulDispatch::Gemm(problem.select_plan());}- if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {- return match route {- QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(- GemvSpecialization::select_tile(- shape,- gemm_b_layout,- weights_data_type,- input_data_type,- output_data_type,- tile,- )- .expect("typed QMV route must contain a legal GEMV tile"),- ),- QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),+ // Small M, one rule for every chip: batched GEMV for M=2..3 (M=2..4 with MXU), the register GEMM above that.+ if all_bf16(weights_data_type, input_data_type, output_data_type) && small_m_weights(shape).is_some() {+ if !supports_mxu && let Some(plan) = problem.small_batch_plan(gemm_b_layout) {+ return MatmulDispatch::Gemm(plan);+ }+ let tile = match (supports_mxu, shape.m) {+ (false, 2..=8) => Some(GemvTile::quantized_output_tile(4, 32, shape.m.min(4), 8, 1)),+ (true, 2..=4) => Some(GemvTile::quantized_output_tile(2, 16, shape.m, 8, 1)),+ _ => None,};+ if let Some(gemv) = tile.and_then(|tile| GemvSpecialization::select_tile(shape, gemm_b_layout, tile)) {+ return MatmulDispatch::Gemv(gemv);+ }}let gemv = GemvSpecialization::select_shape(shape,@@ -122,7 +115,6 @@ impl MatmulMetalKernel {input_data_type,output_data_type,gpu_core_count,- apple_gpu_family,);let plan = problem.select_plan();match gemv {@@ -145,7 +137,6 @@ impl MatmulMetalKernel {Self::choose_dispatch(shape,gemm_b_layout,- &context.device_name,context.gpu_core_count,context.apple_gpu_family,context.supports_mxu,@@ -156,6 +147,14 @@ impl MatmulMetalKernel {}}+fn all_bf16(+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+) -> bool {+ (weights_data_type, input_data_type, output_data_type) == (DataType::BF16, DataType::BF16, DataType::BF16)+}+impl MatmulKernel for MatmulMetalKernel {type Backend = Metal;type Error = MatmulMetalError;@@ -200,7 +199,6 @@ impl MatmulKernel for MatmulMetalKernel {if !context.supports_mxu|| self.input_data_type != DataType::BF16|| self.output_data_type != DataType::BF16- || shape.a_full_precision|| !shape.is_quant()|| shape.params_layout != Some(QuantParamsLayout::GroupOutput)|| !supports_integer_right_operand(shape)@@ -217,29 +215,52 @@ impl MatmulKernel for MatmulMetalKernel {Some(quantization)}- fn select_activation_format(+ /// Load-time: the activation format this linear prepares once (Int8 on MXU, PackedHalf for the small-batch W4+ /// GEMM, else Bf16). `select_activation_format` decides per batch size whether it is used.+ fn activation_preparation(&self,- bf16_shape: &MatmulShape,+ shape: &MatmulShape,weights_layout: GemmBLayout,context: &MetalContext,) -> ActivationFormat {- if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)- || !supports_integer_right_operand(bf16_shape)- || matches!(self.select_dispatch(bf16_shape, weights_layout, context), MatmulDispatch::Gemv(_))- {- return ActivationFormat::Bf16;+ if let Some(quantization) = self.select_activation_quantization(shape, context) {+ return ActivationFormat::Int8(quantization);}- if weights_layout == GemmBLayout::MxuTiled {- return ActivationFormat::Int8;+ match small_batch_format(*shape, weights_layout) {+ Some((format, _))+ if !context.supports_mxu+ && shape.params_layout == Some(QuantParamsLayout::GroupOutput)+ && all_bf16(self.weights_data_type, self.input_data_type, self.output_data_type) =>+ {+ format+ },+ _ => ActivationFormat::Bf16,}+ }- let a8_shape = MatmulShape {- a_full_precision: false,- ..*bf16_shape+ /// Encode-time: `prepared` when this batch size has a kernel for it, else Bf16 (GEMV keeps Bf16 input).+ fn select_activation_format(+ &self,+ bf16_shape: &MatmulShape,+ weights_layout: GemmBLayout,+ prepared: ActivationFormat,+ context: &MetalContext,+ ) -> ActivationFormat {+ let dispatch = |shape: &MatmulShape| self.select_dispatch(shape, weights_layout, context);+ let usable = match prepared {+ ActivationFormat::PackedHalf {+ ..+ } => matches!(+ dispatch(&MatmulShape { a: prepared, ..*bf16_shape }),+ MatmulDispatch::Gemm(plan) if plan.tiling.is_small_batch_variant()+ ),+ ActivationFormat::Int8(_) => matches!(dispatch(bf16_shape), MatmulDispatch::Gemm(_)),+ _ => false,};- match self.select_dispatch(&a8_shape, weights_layout, context) {- MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,- _ => ActivationFormat::Bf16,+ if usable {+ prepared+ } else {+ ActivationFormat::Bf16}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex 1a5803108..58e6b5ce8 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -4,12 +4,9 @@ mod gemv;mod matmul_dispatch;mod matmul_metal_kernel;mod matmul_output_work;-mod qmv;pub use error::MatmulMetalError;pub use gemm::GemmKernel;-#[cfg(test)]-use matmul_dispatch::MatmulDispatch;pub use matmul_metal_kernel::MatmulMetalKernel;use matmul_metal_kernel::supports_integer_right_operand;pub use matmul_output_work::MatmulOutputWork;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rsdeleted file mode 100644index b75f223f9..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs+++ /dev/null@@ -1,12 +0,0 @@-mod routes;--pub(super) use routes::route;--use super::{gemm::GemmPlan, gemv::GemvTile};--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub(super) enum QmvRoute {- Tuned(GemvTile),- MainGemv(GemvTile),- MainGemm(GemmPlan),-}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rsdeleted file mode 100644index 2042c758a..000000000--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs+++ /dev/null@@ -1,239 +0,0 @@-// Qwen3.6-27B W4/ZP and W8/Symmetric, G64, plus Weaver linears on M5 Max.-// Tuned on M1, M2, M2 Pro, M3 Max, M4, M4 Pro, M5 Max.-use metal::MTLGPUFamily;--use super::{- super::{- gemm::{GemmEngine, GemmPlan},- gemv::GemvTile,- },- QmvRoute,-};-use crate::backends::{- common::{gpu_types::gemm::GemmBPrologueKind, kernel::matmul::MatmulShape},- metal::gpu_types::gemm::GemmTiling,-};--const DOWN: u8 = 1 << 0;-const GATE: u8 = 1 << 1;-const GATE_UP: u8 = 1 << 2;-const PROJECTION_IN: u8 = 1 << 3;-const PROJECTION_OUT: u8 = 1 << 4;-const READOUT: u8 = 1 << 5;-const WEAVER: u8 = 1 << 6;--#[derive(Clone, Copy)]-struct RouteRow {- device_name: &'static str,- apple_gpu_family: MTLGPUFamily,- bits: u32,- group: u32,- m: u32,- shapes: u8,- route: QmvRoute,-}--const fn shape(- n: u32,- k: u32,-) -> u8 {- match (n, k) {- (5120, 17408) => DOWN,- (6144, 5120) => GATE,- (34816, 5120) => GATE_UP,- (16480, 5120) => PROJECTION_IN,- (5120, 6144) => PROJECTION_OUT,- (248320, 5120) => READOUT,- (5120 | 6144, 2048) => WEAVER,- _ => 0,- }-}--#[rustfmt::skip]-macro_rules! tuned { ($input:literal, $output:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::Tuned(GemvTile::quantized_output_tile($simdgroups, $output, $input, $lanes, $group_lanes)) }; }-#[rustfmt::skip]-macro_rules! main_gemv { ($input:literal, $results_per_simdgroup:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::MainGemv(GemvTile::quantized($simdgroups, $results_per_simdgroup, $input, $lanes, $group_lanes)) }; }-#[rustfmt::skip]-macro_rules! main_gemm { ($engine:ident, $tiling:ident, $split:literal) => { QmvRoute::MainGemm(GemmPlan { engine: GemmEngine::$engine, tiling: GemmTiling::$tiling, split_k: $split }) }; }-#[rustfmt::skip]-macro_rules! qmv_format { (w4_zp_g64) => { (4, 64) }; (w8_sym_g64) => { (8, 64) }; }-#[rustfmt::skip]-macro_rules! measured_device_name { (M1) => { "Apple M1" }; (M2) => { "Apple M2" }; (M2Pro) => { "Apple M2 Pro" }; (M3Max) => { "Apple M3 Max" }; (M4) => { "Apple M4" }; (M4Pro) => { "Apple M4 Pro" }; (M5Max) => { "Apple M5 Max" }; }-#[rustfmt::skip]-macro_rules! measured_apple_gpu_family { (M1) => { MTLGPUFamily::Apple7 }; (M2) => { MTLGPUFamily::Apple8 }; (M2Pro) => { MTLGPUFamily::Apple8 }; (M3Max) => { MTLGPUFamily::Apple9 }; (M4) => { MTLGPUFamily::Apple9 }; (M4Pro) => { MTLGPUFamily::Apple9 }; (M5Max) => { MTLGPUFamily::Apple10 }; }-#[rustfmt::skip]-macro_rules! row { ($device:ident, $format:ident, $m:literal, $shapes:expr, $route:expr) => { RouteRow { device_name: measured_device_name!($device), apple_gpu_family: measured_apple_gpu_family!($device), bits: qmv_format!($format).0, group: qmv_format!($format).1, m: $m, shapes: $shapes, route: $route } }; }--#[rustfmt::skip]-const ROWS: &[RouteRow] = &[- row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 16, 1, 4)),- row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M1, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M1, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M1, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 2, PROJECTION_IN, tuned!(2, 8, 8, 1, 2)),- row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 2, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M2, w8_sym_g64, 2, DOWN, tuned!(2, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M2, w8_sym_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 3, PROJECTION_OUT, tuned!(3, 16, 16, 1, 4)),- row!(M2Pro, w4_zp_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),- row!(M2Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),- row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M2Pro, w8_sym_g64, 3, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M2Pro, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 4, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M2Pro, w8_sym_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M2Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M3Max, w4_zp_g64, 2, DOWN | GATE | PROJECTION_OUT, tuned!(2, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 3, DOWN | GATE | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 4, GATE_UP | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 5, GATE_UP, tuned!(5, 16, 16, 1, 4)),- row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M3Max, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M3Max, w8_sym_g64, 3, DOWN | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 4, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),- row!(M3Max, w8_sym_g64, 4, DOWN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M3Max, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M4, w4_zp_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M4, w4_zp_g64, 4, DOWN | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),- row!(M4, w4_zp_g64, 5, DOWN | GATE_UP | PROJECTION_IN | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT, tuned!(5, 16, 16, 1, 4)),- row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),- row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),- row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4Pro, w8_sym_g64, 2, READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),- row!(M4Pro, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M4Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT | WEAVER, tuned!(4, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M5Max, w4_zp_g64, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w4_zp_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w4_zp_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),- row!(M5Max, w4_zp_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),- row!(M5Max, w4_zp_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w4_zp_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w4_zp_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),- row!(M5Max, w4_zp_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),- row!(M5Max, w8_sym_g64, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),- row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w8_sym_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),- row!(M5Max, w8_sym_g64, 6, GATE_UP | PROJECTION_IN | READOUT, tuned!(6, 16, 8, 1, 2)),- row!(M5Max, w8_sym_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),- row!(M5Max, w8_sym_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),- row!(M5Max, w8_sym_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),-];--fn qmv_format(shape: &MatmulShape) -> Option<(u32, u32)> {- match (shape.b_prologue, shape.b_bits, shape.b_group_size) {- (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(64)) => Some((4, 64)),- (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(64)) => Some((8, 64)),- _ => None,- }-}--pub fn route(- device_name: &str,- apple_gpu_family: MTLGPUFamily,- supports_mxu: bool,- shape: &MatmulShape,- all_bf16: bool,-) -> Option<QmvRoute> {- if !all_bf16- || shape.m < 2- || shape.m > 7- || !shape.a_full_precision- || !shape.b_transpose- || shape.b_leading_dimension.is_some()- || shape.gathered- {- return None;- }- let (bits, group) = qmv_format(shape)?;- let mask = self::shape(shape.n, shape.k);- if mask == 0 {- return None;- }- let matches =- |row: &&RouteRow| row.bits == bits && row.group == group && row.m == shape.m && row.shapes & mask != 0;- let route =- ROWS.iter().filter(matches).find(|row| row.device_name == device_name).map(|row| row.route).or_else(|| {- let same_family = |row: &&RouteRow| row.apple_gpu_family == apple_gpu_family;- let mut routes = ROWS.iter().filter(matches).filter(same_family).map(|row| row.route);- let route = routes.next()?;- routes.all(|candidate| candidate == route).then_some(route)- });- route.filter(|route| {- !matches!(- route,- QmvRoute::MainGemm(GemmPlan {- engine: GemmEngine::Mxu,- ..- })- ) || supports_mxu- })-}--#[cfg(test)]-#[path = "../../../../../../unit/backends/metal/kernel/matmul/qmv/routes_test.rs"]-mod tests;diff --git a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slangindex 70d933335..f1f147bc7 100644--- a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang@@ -6,7 +6,7 @@ import generated.activation_transform;[[Variants("BiasT", "float, BFloat16")]][[Public]] [[Kernel]] func ActivationTransform<T, BiasT>([[Optional("!in_place")]] StructuredBuffer<T> input,- [[Optional("!(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<T> fp_out,+ [[Optional("(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)")]] RWStructuredBuffer<T> fp_out,[[Optional("has_bias")]] StructuredBuffer<BiasT> bias,[[Optional("(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int8_t> q_out,[[Optional("(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<float> scales_out,@@ -14,6 +14,7 @@ import generated.activation_transform;StructuredBuffer<int> rht_factors,uint batch_size,uint element_count,+ [[Optional("(ops == ActivationTransformOp::PackHalf)")]] RWStructuredBuffer<uint> packed_half_out,[[Specialize]] ActivationTransformOp ops,[[Specialize]] bool grouped_by_weight_nibble,[[Specialize]] bool in_place,diff --git a/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang b/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slangindex 25889a958..3f12193ef 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang@@ -7,7 +7,7 @@ import generated.gated_act_mul;[[Public]] [[Kernel]] func GatedActMul<T>(StructuredBuffer<T> act_operand,[[Optional("!interleaved")]] StructuredBuffer<T> value_operand,- [[Optional("!(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<T> fp_out,+ [[Optional("(ops == GatedActMulOp::FullPrecision)")]] RWStructuredBuffer<T> fp_out,[[Optional("(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int8_t> q_out,[[Optional("(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<float> scales_out,[[Optional("(ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int> group_sums_out,@@ -22,6 +22,7 @@ import generated.gated_act_mul;[[Optional("clip_gate")]] float gate_clip_max,[[Optional("clip_value")]] float value_clip_min,[[Optional("clip_value")]] float value_clip_max,+ [[Optional("(ops == GatedActMulOp::PackHalf)")]] RWStructuredBuffer<uint> packed_half_out,[[Specialize]] GatedActMulOp ops,[[Specialize]] bool grouped_by_weight_nibble,[[Specialize]] bool interleaved,diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rsindex c020b69c8..6ccb1a82f 100644--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs@@ -4,7 +4,10 @@ use super::{LinearInput, LinearInputPreparation};use crate::{backends::common::{Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding,- kernel::{ActivationTransform, matmul::ActivationFormat},+ kernel::{+ ActivationTransform,+ matmul::{ActivationFormat, MatmulA, packed_half_bytes},+ },},data_type::DataType,};@@ -24,11 +27,11 @@ impl<B: Backend> InputRht<B> {) -> Result<Self, B::Error> {let LinearInputPreparation {rht_signs,- activation_quantization,+ activation_format,} = preparation;let rht = ActivationTransform::input_rht(context, data_type, in_place)?;- let quantizer = activation_quantization- .map(|quantization| ActivationTransform::quantize(context, data_type, quantization))+ let quantizer = (activation_format != ActivationFormat::Bf16)+ .then(|| ActivationTransform::quantize(context, data_type, activation_format)).transpose()?;Ok(Self {@@ -45,14 +48,26 @@ impl<B: Backend> InputRht<B> {format: ActivationFormat,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<LinearInput<B>, B::Error> {- if format == ActivationFormat::Int8- && let Some(quantizer) = &self.quantizer- {+ if let Some(quantizer) = self.quantizer.as_ref().filter(|quantizer| quantizer.format() == format) {let input_dim = self.input_dim();- let groups_per_row = input_dim.div_ceil(quantizer.scale_group_size());+ if let ActivationFormat::PackedHalf {+ group_size,+ } = format+ {+ let mut values = command_buffer.allocate_scratch(packed_half_bytes(input_dim, group_size))?;+ quantizer.encode_pack_half(input, &mut values, &self.rht_signs, batch_dim, input_dim, command_buffer);+ return Ok(MatmulA::PackedHalf {+ values,+ group_size,+ });+ }+ let ActivationFormat::Int8(quantization) = format else {+ unreachable!()+ };+ let groups_per_row = input_dim.div_ceil(quantization.scale_group_size());let mut values = command_buffer.allocate_scratch_for_shape(&[batch_dim, input_dim], DataType::I8)?;let mut scales = command_buffer.allocate_scratch_for_shape(&[batch_dim, groups_per_row], DataType::F32)?;- let mut group_sums = quantizer+ let mut group_sums = quantization.sum_group_size().map(|group_size| {command_buffer@@ -70,19 +85,18 @@ impl<B: Backend> InputRht<B> {command_buffer,);- return Ok(LinearInput::Int8Symmetric {+ return Ok(MatmulA::Int8Symmetric {values,scales,group_sums,- scale_group_size: quantizer.scale_group_size(),- code_layout: quantizer.code_layout(),+ quantization,});}let input_dim = self.input_dim();let mut transformed = command_buffer.allocate_scratch(input.size())?;self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, command_buffer);- Ok(LinearInput::FullPrecision(transformed))+ Ok(MatmulA::full_precision(transformed))}pub(super) fn prepare_in_place(@@ -92,13 +106,13 @@ impl<B: Backend> InputRht<B> {format: ActivationFormat,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<LinearInput<B>, B::Error> {- if format == ActivationFormat::Int8 && self.quantizer.is_some() {+ if self.quantizer.as_ref().is_some_and(|kernel| kernel.format() == format) {return self.prepare(&input, batch_dim, format, command_buffer);}let input_dim = self.input_dim();self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, command_buffer);- Ok(LinearInput::FullPrecision(input))+ Ok(MatmulA::full_precision(input))}fn input_dim(&self) -> u32 {diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex b86a60475..9e2679be5 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -6,7 +6,7 @@ use crate::{backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding,kernel::{- ActivationQuantization, Kernels,+ Kernels,matmul::{ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,MatmulShape,@@ -46,6 +46,7 @@ pub struct LinearMatmul<B: Backend> {input_dim: u32,output_dim: u32,output_data_type: DataType,+ prepared_activation_format: ActivationFormat,}async fn load_biases<B: Backend>(@@ -110,17 +111,23 @@ impl<B: Backend> LinearMatmul<B> {input_dim,output_dim,output_data_type,+ prepared_activation_format: ActivationFormat::Bf16,})}- pub(super) fn optimize_weights_layout(+ pub(super) fn prepare_activation(&mut self,context: &B::Context,- ) -> Result<Option<ActivationQuantization>, B::Error> {- let shape = self.matmul_shape(1, false);- let kernel = self.kernel.lock();- self.matrix.optimize_weights_layout(|weights| kernel.optimize_weights_layout(&shape, weights, context))?;- Ok(kernel.select_activation_quantization(&self.matmul_shape(1, false), context))+ ) -> Result<ActivationFormat, B::Error> {+ {+ let shape = self.matmul_shape(1, ActivationFormat::Bf16);+ let kernel = self.kernel.lock();+ self.matrix.optimize_weights_layout(|weights| kernel.optimize_weights_layout(&shape, weights, context))?;+ }+ let shape = self.matmul_shape(1, ActivationFormat::Bf16);+ self.prepared_activation_format =+ self.kernel.lock().activation_preparation(&shape, self.matmul_b().weights_layout(), context);+ Ok(self.prepared_activation_format)}pub(super) fn encode_with_a(@@ -159,7 +166,7 @@ impl<B: Backend> LinearMatmul<B> {fn matmul_shape(&self,batch_dim: u32,- a_full_precision: bool,+ a: ActivationFormat,) -> MatmulShape {let b = self.matmul_b();MatmulShape {@@ -173,7 +180,7 @@ impl<B: Backend> LinearMatmul<B> {b_bits: b.bits_per_b(),b_group_size: b.group_size(),signed_codes: b.signed_codes(),- a_full_precision,+ a,gathered: false,params_layout: b.quant_params_layout(),d_transform: self.d_ops().mask(),@@ -200,7 +207,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {- self.encode_with_a(input.as_matmul_a(), batch_dim, None::<Gather<&B::ScratchBuffer>>, command_buffer)+ self.encode_with_a(input.as_ref(), batch_dim, None::<Gather<&B::ScratchBuffer>>, command_buffer)}fn select_activation_format(@@ -208,7 +215,12 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {batch_dim: u32,context: &B::Context,) -> ActivationFormat {- let bf16_shape = self.matmul_shape(batch_dim, true);- self.kernel.lock().select_activation_format(&bf16_shape, self.matmul_b().weights_layout(), context)+ let shape = self.matmul_shape(batch_dim, ActivationFormat::Bf16);+ self.kernel.lock().select_activation_format(+ &shape,+ self.matmul_b().weights_layout(),+ self.prepared_activation_format,+ context,+ )}}diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rsindex 73f52ef53..ad96f26b5 100644--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs@@ -14,10 +14,7 @@ use crate::{backends::common::{Backend, BufferRef, CommandBuffer, Kernels,gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,- kernel::{- ActivationQuantization,- matmul::{ActivationFormat, Int8CodeLayout, MatmulA, MatmulKernel},- },+ kernel::matmul::{ActivationFormat, MatmulA, MatmulKernel},},config::weight_matrix::{AnyWeightMatrixSpec,@@ -42,7 +39,7 @@ pub trait Linear<B: Backend>: Send + Sync {batch_dim: u32,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {- self.encode_input(LinearInput::FullPrecision(input), batch_dim, command_buffer)+ self.encode_input(MatmulA::full_precision(input), batch_dim, command_buffer)}fn encode_input(@@ -61,49 +58,16 @@ pub trait Linear<B: Backend>: Send + Sync {}}-pub enum LinearInput<B: Backend> {- FullPrecision(B::ScratchBuffer),- Int8Symmetric {- values: B::ScratchBuffer,- scales: B::ScratchBuffer,- group_sums: Option<B::ScratchBuffer>,- scale_group_size: u32,- code_layout: Int8CodeLayout,- },-}+pub type LinearInput<B> = MatmulA<<B as Backend>::ScratchBuffer>;pub struct Gather<T: BufferRef> {pub indices: T,pub output_dim: u32,}-impl<B: Backend> LinearInput<B> {- fn as_matmul_a(&self) -> MatmulA<&B::ScratchBuffer> {- match self {- Self::FullPrecision(values) => MatmulA::FullPrecision {- values,- offset: 0,- },- Self::Int8Symmetric {- values,- scales,- group_sums,- scale_group_size,- code_layout,- } => MatmulA::Int8Symmetric {- values,- scales,- group_sums: group_sums.as_ref(),- scale_group_size: *scale_group_size,- code_layout: *code_layout,- },- }- }-}-pub struct LinearInputPreparation<B: Backend> {pub rht_signs: B::GlobalGpuBuffer,- pub activation_quantization: Option<ActivationQuantization>,+ pub activation_format: ActivationFormat,}#[derive(Debug, Error)]diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rsindex f26308b16..6fdba1bde 100644--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs@@ -3,7 +3,9 @@ use thiserror::Error;use crate::{backends::common::{- Backend, CommandBuffer, CommandBufferEncoding, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ Backend, CommandBuffer, CommandBufferEncoding,+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,+ kernel::{ActivationFormat, matmul::MatmulA},},config::weight_matrix::{AnyWeightMatrixSpec,@@ -62,7 +64,7 @@ impl<B: Backend> RHTLinearWrapper<B> {return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));}- let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(+ let (rht_signs, inner_linear, activation_format) = Self::load_inner_with_output_rht(input_dimension,output_dimension,weights_data_type,@@ -73,13 +75,11 @@ impl<B: Backend> RHTLinearWrapper<B> {context,).await?;- let activation_quantization =- inner_linear.optimize_weights_layout(context).map_err(RHTLinearWrapperError::BackendError)?;Self::build_self_contained(input_data_type,LinearInputPreparation {rht_signs,- activation_quantization,+ activation_format,},inner_linear,context,@@ -103,7 +103,7 @@ impl<B: Backend> RHTLinearWrapper<B> {return Ok(None);}- let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(+ let (rht_signs, inner_linear, activation_format) = Self::load_inner_with_output_rht(input_dimension,output_dimension,weights_data_type,@@ -116,11 +116,9 @@ impl<B: Backend> RHTLinearWrapper<B> {.await?;let input_preparation = LinearInputPreparation {rht_signs,- activation_quantization: inner_linear- .optimize_weights_layout(context)- .map_err(RHTLinearWrapperError::BackendError)?,+ activation_format,};- if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {+ if input_preparation.activation_format != ActivationFormat::Bf16 && !allow_prequantized_activation {let wrapper = Self::build_self_contained(input_data_type, input_preparation, inner_linear, context)?;Ok(Some((Box::new(wrapper), None)))} else {@@ -137,7 +135,7 @@ impl<B: Backend> RHTLinearWrapper<B> {has_biases: bool,parameter_tree: &ParameterTree<'_, B>,context: &B::Context,- ) -> Result<(B::GlobalGpuBuffer, LinearMatmul<B>), RHTLinearWrapperError<B>> {+ ) -> Result<(B::GlobalGpuBuffer, LinearMatmul<B>, ActivationFormat), RHTLinearWrapperError<B>> {let weights_tree = parameter_tree.subtree("weights");let quantized_weights_tree = weights_tree.subtree("quantized");let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;@@ -145,7 +143,7 @@ impl<B: Backend> RHTLinearWrapper<B> {weights_tree.read_buffer("incoherence_signs.input_signs", &[input_dimension], DataType::I32).await?;let output_factors =weights_tree.read_buffer("incoherence_signs.output_signs", &[output_dimension], DataType::I32).await?;- let inner_linear = LinearMatmul::load(+ let mut inner_linear = LinearMatmul::load(input_dimension,output_dimension,weights_data_type,@@ -158,7 +156,9 @@ impl<B: Backend> RHTLinearWrapper<B> {context,).await?;- Ok((rht_signs, inner_linear))+ let activation_format =+ inner_linear.prepare_activation(context).map_err(RHTLinearWrapperError::BackendError)?;+ Ok((rht_signs, inner_linear, activation_format))}fn build_self_contained(@@ -185,7 +185,10 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {let input = match input {- LinearInput::FullPrecision(input) => input,+ MatmulA::FullPrecision {+ values: input,+ ..+ } => input,input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),};let command_buffer = &mut *command_buffer.debug_group("linear (rht)");@@ -196,7 +199,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {.prepare_in_place(input, batch_dim, format, command_buffer).map_err(LinearEncodeError::Backend)?;let output = self.inner_linear.encode_with_a(- input.as_matmul_a(),+ input.as_ref(),batch_dim,None::<Gather<&B::ScratchBuffer>>,command_buffer,diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rsindex 33bf030f8..e474d2c10 100644--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs@@ -60,11 +60,10 @@ impl<B: Backend> UntiedReadout<B> {.await?;let rht_signs =tree.subtree("incoherence_signs").read_buffer("input_signs", &[model_dim], DataType::I32).await?;+ let activation_format = linear.prepare_activation(context).map_err(LinearMatmulError::BackendError)?;let preparation = LinearInputPreparation {rht_signs,- activation_quantization: linear- .optimize_weights_layout(context)- .map_err(LinearMatmulError::BackendError)?,+ activation_format,};let input_rht = InputRht::new(data_type, preparation, /* in_place */ false, context).map_err(LinearMatmulError::BackendError)?;@@ -83,10 +82,7 @@ impl<B: Backend> UntiedReadout<B> {command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {let Some(input_rht) = &self.input_rht else {- let a = MatmulA::FullPrecision {- values: input,- offset: 0,- };+ let a = MatmulA::full_precision(input);return self.linear.encode_with_a(a, batch_dim, gather, command_buffer);};@@ -96,6 +92,6 @@ impl<B: Backend> UntiedReadout<B> {self.linear.select_activation_format(batch_dim, command_buffer.context())};let input = input_rht.prepare(input, batch_dim, format, command_buffer).map_err(LinearEncodeError::Backend)?;- self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)+ self.linear.encode_with_a(input.as_ref(), batch_dim, gather, command_buffer)}}diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rsindex a97d1abaa..2a7847bc0 100644--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs@@ -2,7 +2,10 @@ use crate::{backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding,gpu_types::activation_type::ActivationType,- kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::ActivationFormat},+ kernel::{+ GatedActMul, GatedActMulSettings,+ matmul::{ActivationFormat, MatmulA, packed_half_bytes},+ },},config::{activation::AnyActivation, clipping::ClippingBounds},data_type::DataType,@@ -15,7 +18,6 @@ pub struct MlpGateActMulEncodable<B: Backend> {hidden_dim: u32,data_type: DataType,hadamard_factors: Option<B::GlobalGpuBuffer>,- activation_quantization: Option<ActivationQuantization>,quantized_kernel: Option<GatedActMul<B>>,}@@ -29,16 +31,17 @@ impl<B: Backend> MlpGateActMulEncodable<B> {input_preparation: Option<LinearInputPreparation<B>>,context: &B::Context,) -> Result<Self, B::Error> {- let (hadamard_factors, activation_quantization) = input_preparation- .map_or((None, None), |preparation| (Some(preparation.rht_signs), preparation.activation_quantization));+ let (hadamard_factors, format) = input_preparation.map_or((None, ActivationFormat::Bf16), |preparation| {+ (Some(preparation.rht_signs), preparation.activation_format)+ });let settings = GatedActMulSettings {activation_alpha: activation.custom_alpha(),gate_clipping,value_clipping,};let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some(), settings)?;- let quantized_kernel = activation_quantization- .map(|quantization| GatedActMul::quantized(context, data_type, quantization, settings))+ let quantized_kernel = (format != ActivationFormat::Bf16)+ .then(|| GatedActMul::quantized(context, data_type, format, settings)).transpose()?;Ok(Self {fp_kernel,@@ -46,7 +49,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {hidden_dim,data_type,hadamard_factors,- activation_quantization,quantized_kernel,})}@@ -61,10 +63,31 @@ impl<B: Backend> MlpGateActMulEncodable<B> {if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}- let input = if act_format == ActivationFormat::Int8- && let Some(quantization) = self.activation_quantization- {- let kernel = self.quantized_kernel.as_ref().expect("INT8 input requires a quantized gate kernel");++ if let Some(kernel) = self.quantized_kernel.as_ref().filter(|kernel| kernel.format() == act_format) {+ let hadamard_factors = self.hadamard_factors.as_ref().expect("quantized input requires RHT factors");+ if let ActivationFormat::PackedHalf {+ group_size,+ } = act_format+ {+ let mut values = command_buffer.allocate_scratch(packed_half_bytes(self.hidden_dim, group_size))?;+ kernel.encode_pack_half(+ fused_up,+ &mut values,+ hadamard_factors,+ self.hidden_dim,+ batch_dim,+ self.activation.act_type(),+ command_buffer,+ );+ return Ok(MatmulA::PackedHalf {+ values,+ group_size,+ });+ }+ let ActivationFormat::Int8(quantization) = act_format else {+ unreachable!()+ };let mut values = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], DataType::I8)?;let mut scales = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim.div_ceil(quantization.scale_group_size())],@@ -82,37 +105,33 @@ impl<B: Backend> MlpGateActMulEncodable<B> {&mut values,&mut scales,group_sums.as_mut(),- self.hadamard_factors.as_ref().expect("INT8 input requires RHT factors"),+ hadamard_factors,self.hidden_dim,batch_dim,self.activation.act_type(),command_buffer,);- LinearInput::Int8Symmetric {+ return Ok(MatmulA::Int8Symmetric {values,scales,group_sums,- scale_group_size: quantization.scale_group_size(),- code_layout: quantization.code_layout(),- }- } else {- let mut hidden =- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;- self.fp_kernel.encode_fp(- fused_up,- None::<&B::ScratchBuffer>,- &mut hidden,- self.hadamard_factors.as_ref(),- self.hidden_dim,- batch_dim,- 0,- 0,- self.activation.act_type(),- command_buffer,- );- LinearInput::FullPrecision(hidden)- };+ quantization,+ });+ }- Ok(input)+ let mut hidden = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;+ self.fp_kernel.encode_fp(+ fused_up,+ None::<&B::ScratchBuffer>,+ &mut hidden,+ self.hadamard_factors.as_ref(),+ self.hidden_dim,+ batch_dim,+ 0,+ 0,+ self.activation.act_type(),+ command_buffer,+ );+ Ok(MatmulA::full_precision(hidden))}}diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rsindex 4196fee4d..e6d5c858f 100644--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs@@ -157,7 +157,7 @@ mod quantize {backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- kernel::{ActivationQuantization, ActivationTransform, matmul::Int8CodeLayout},+ kernel::{ActivationFormat, ActivationQuantization, ActivationTransform, matmul::Int8CodeLayout},},cpu::Cpu,},@@ -187,13 +187,15 @@ mod quantize {let kernel = ActivationTransform::<B>::quantize(context.as_ref(),DataType::F32,- ActivationQuantization::new(- scale_group_size,- sum_group_size.unwrap_or(scale_group_size),- emit_group_sums,- code_layout,- )- .expect("supported activation quantization"),+ ActivationFormat::Int8(+ ActivationQuantization::new(+ scale_group_size,+ sum_group_size.unwrap_or(scale_group_size),+ emit_group_sums,+ code_layout,+ )+ .expect("supported activation quantization"),+ ),).expect("quantize transform");let mut command_buffer = context.as_ref().create_command_buffer(None, None).expect("command buffer");diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rsindex cceaec422..c7adf46f4 100644--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs@@ -10,7 +10,9 @@ use crate::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,gpu_types::activation_type::ActivationType,- kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::Int8CodeLayout},+ kernel::{+ ActivationFormat, ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::Int8CodeLayout,+ },},cpu::Cpu,},@@ -125,8 +127,10 @@ fn run_nibble_grouped_quantized<B: Backend>(input: &InterleavedInput<bf16>) -> (let kernel = GatedActMul::<B>::quantized(&context,DataType::BF16,- ActivationQuantization::new(128, 128, false, Int8CodeLayout::GroupedByNibble)- .expect("supported activation quantization"),+ ActivationFormat::Int8(+ ActivationQuantization::new(128, 128, false, Int8CodeLayout::GroupedByNibble)+ .expect("supported activation quantization"),+ ),GatedActMulSettings::default(),).expect("create quantized GatedActMul");diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rsindex 236352bf0..86a3ed3fa 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs@@ -13,7 +13,7 @@ use crate::{quantization_method::QuantizationMethod,},kernel::{- ActivationQuantization, ActivationTransform, Kernels,+ ActivationFormat, ActivationQuantization, ActivationTransform, Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,@@ -135,10 +135,7 @@ impl BenchmarkData {&'a <Metal as Backend>::GlobalGpuBuffer,> {MatmulArguments {- a: MatmulA::FullPrecision {- values: &self.a_working,- offset: 0,- },+ a: MatmulA::full_precision(&self.a_working),b: MatmulB::Quantized(QuantizedB {codes: &self.unsigned_weights,scales: &self.weight_scales,@@ -171,6 +168,9 @@ fn encode_step() {match path {BenchPath::A8GemmMxu => {+ let ActivationFormat::Int8(quantization) = prepare.format() else {+ unreachable!("A8 benchmark requires Int8 activation preparation")+ };prepare.encode_quantize(&data.activations,&mut data.a_int8,@@ -192,9 +192,7 @@ fn encode_step(values: &data.a_int8,scales: &data.a_scales,group_sums: None,- scale_group_size: 128,- code_layout: Int8CodeLayout::for_right_bits(DataType::from(data.mode).size_in_bits() as u32)- .expect("W4/W8 benchmark"),+ quantization,},b: MatmulB::Quantized(QuantizedB {codes: &data.a8_weights,@@ -287,8 +285,10 @@ fn bench_a8w(c: &mut Criterion) {let prepare = ActivationTransform::<Metal>::quantize(&context,DataType::BF16,- ActivationQuantization::new(128, 128, false, Int8CodeLayout::for_right_bits(bits).expect("W8 benchmark"))- .expect("supported activation quantization"),+ ActivationFormat::Int8(+ ActivationQuantization::new(128, 128, false, Int8CodeLayout::for_right_bits(bits).expect("W8 benchmark"))+ .expect("supported activation quantization"),+ ),).expect("prepare kernel");bench_bits(c, &context, &prepare, &hadamard, bits);diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex e53936048..913832e7e 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -3,38 +3,18 @@ use metal::MTLGPUFamily;use uzu_engine_macros::uzu_test;use super::*;-use crate::backends::{- common::{- Context,- gpu_types::gemm::GemmDTransform,- kernel::matmul::{MatmulKernel, QuantParamsLayout},+use crate::{+ backends::{+ common::{+ Context,+ gpu_types::gemm::GemmDTransform,+ kernel::matmul::{MatmulKernel, QuantParamsLayout},+ },+ metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},},- metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},+ tests::matmul::{a8_format, matmul_shape as shape, quantized_matmul_shape},};-fn shape(- m: u32,- n: u32,- k: u32,-) -> MatmulShape {- MatmulShape {- m,- n,- k,- b_transpose: true,- b_leading_dimension: None,- b_prologue: GemmBPrologueKind::FullPrecision,- b_is_trellis: false,- b_bits: None,- b_group_size: None,- signed_codes: false,- a_full_precision: true,- gathered: false,- params_layout: Some(QuantParamsLayout::OutputGroup),- d_transform: GemmDTransform::empty(),- }-}-fn quant(mut shape: MatmulShape) -> MatmulShape {shape.b_prologue = GemmBPrologueKind::ScaleSymmetricDequant;shape.b_bits = Some(4);@@ -118,7 +98,7 @@ fn selection_fallbacks_and_split_k_are_preserved() {let mut invalid_layout = quant(shape(64, 4096, 4096));invalid_layout.b_transpose = false;assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Simdgroup);- invalid_layout.a_full_precision = false;+ invalid_layout.a = a8_format();assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Mxu);assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);@@ -129,14 +109,14 @@ fn selection_fallbacks_and_split_k_are_preserved() {(8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),] {let mut a8 = quant(shape(m, n, k));- a8.a_full_precision = false;+ a8.a = a8_format();let plan = problem(a8, DataType::BF16).select_plan();assert_eq!(plan.tiling, expected_tiling);assert_eq!(plan.split_k, expected_split_k);}let mut biased = quant(shape(16, 4096, 4096));- biased.a_full_precision = false;+ biased.a = a8_format();biased.d_transform = GemmDTransform::BIAS;assert_eq!(GemmProblem::new(biased, DataType::BF16, DataType::F32, true, MTLGPUFamily::Apple7).select_plan().split_k,@@ -166,7 +146,7 @@ fn trellis_plan_matches_projection_cases() {(1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),] {let mut trellis_shape = shape(m, n, k);- trellis_shape.a_full_precision = false;+ trellis_shape.a = ActivationFormat::Trellis;trellis_shape.b_is_trellis = true;let plan = problem(trellis_shape, DataType::BF16).select_plan();assert_eq!((plan.engine, plan.tiling, plan.split_k), (GemmEngine::Mxu, tiling, split_k), "M {m} N {n} K {k}");@@ -224,22 +204,7 @@ fn int8_activations_require_mxu_tiled_weights() {(64, 20480, 8, true),(64, 20608, 8, false),] {- let shape = MatmulShape {- b_prologue: if bits == 4 {- GemmBPrologueKind::ScaleZeroPointDequant- } else {- GemmBPrologueKind::ScaleSymmetricDequant- },- b_bits: Some(bits),- signed_codes: bits == 8,- a_full_precision: false,- params_layout: Some(QuantParamsLayout::GroupOutput),- ..quant(shape(16, n, k))- };- assert_eq!(- kernel.select_activation_quantization(&shape, &context).is_some(),- expected,- "n={n} k={k} bits={bits}"- );+ let shape = quantized_matmul_shape(16, n, k, bits, 64);+ assert_eq!(kernel.select_activation_quantization(&shape, &context).is_some(), expected, "n={n} k={k} W{bits}");}}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rsindex 6072d4322..da07743a8 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs@@ -1,19 +1,15 @@-use metal::MTLGPUFamily;use uzu_engine_macros::uzu_test;-use super::*;+use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile, fp_tile, quantized_tile, tile};use crate::{backends::{- common::{- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},- kernel::matmul::{MatmulShape, QuantParamsLayout},- },+ common::kernel::matmul::{MatmulShape, QuantParamsLayout},metal::gpu_types::gemm::GemmBLayout,},data_type::DataType,+ tests::matmul::quantized_matmul_shape as shape,};-const SMALL_GPU_CORE_COUNT: u32 = 29;const LARGE_GPU_CORE_COUNT: u32 = 30;fn qtile(@@ -26,121 +22,52 @@ fn qtile(#[uzu_test]fn fp_policy_cases() {let cases = [- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 12288, 1536, true, tile(8, 8, 1)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 12288, 1536, false, tile(8, 1, 1)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, true, tile(8, 1, 1)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, true, tile(8, 2, 1)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 8, 12288, 1536, true, tile(8, 1, 4)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 12288, true, tile(8, 8, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 262144, 1536, true, tile(8, 1, 4)),+ (1, 12288, 1536, true, tile(8, 8, 1)),+ (1, 12288, 1536, false, tile(8, 1, 1)),+ (1, 1536, 256, true, tile(8, 2, 1)),+ (8, 12288, 1536, true, tile(8, 1, 4)),+ (1, 1536, 12288, true, tile(8, 8, 1)),+ (1, 262144, 1536, true, tile(8, 8, 1)),];- for (gpu_core_count, apple_gpu_family, m, n, k, aligned, expected) in cases {- assert_eq!(fp_tile(gpu_core_count, apple_gpu_family, m, n, k, aligned), expected);+ for (m, n, k, aligned, expected) in cases {+ assert_eq!(fp_tile(m, n, k, aligned), expected, "m={m} n={n} k={k} aligned={aligned}");}}#[uzu_test]fn quant_policy_cases() {let cases = [- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 256, 1536, 4, qtile(2, 1)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 262144, 1536, 4, qtile(8, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, 4, qtile(4, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 1, 1536, 256, 4, qtile(8, 4)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 1, 2048, 1536, 4, qtile(8, 2)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 256, 1536, 4, qtile(8, 2)),- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 1536, 256, 4, qtile(4, 8)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 2, 2048, 1536, 4, qtile(8, 4)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 5120, 5120, 4, qtile(8, 4)),- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2048, 1536, 8, GemvTile::quantized(8, 4, 1, 32, 4)),+ (1, 256, 4, qtile(8, 4)),+ (1, 262144, 4, qtile(8, 4)),+ (2, 2048, 4, qtile(8, 4)),+ (4, 5120, 4, qtile(8, 4)),+ (1, 2048, 8, GemvTile::quantized(8, 4, 1, 32, 4)),];- for (gpu_core_count, apple_gpu_family, m, n, k, bits, expected) in cases {- let actual =- quantized_tile(gpu_core_count, apple_gpu_family, bits, 32, m, n, k, true).expect("quantized route");- assert_eq!(- actual, expected,- "m={m} n={n} k={k} bits={bits} gpu_core_count={gpu_core_count} apple_gpu_family={apple_gpu_family:?}"- );+ for (m, n, bits, expected) in cases {+ let actual = quantized_tile(bits, 32, m, n, true).expect("quantized route");+ assert_eq!(actual, expected, "m={m} n={n} bits={bits}");}}-#[uzu_test]-fn gpu_core_count_boundary_selects_large_policy() {- let small = quantized(SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("small GPU route");- let large = quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("large GPU route");-- assert_eq!(small, qtile(4, 2));- assert_eq!(large, qtile(2, 1));-}--fn quantized(- gpu_core_count: u32,- apple_gpu_family: MTLGPUFamily,- bits: u32,- group: u32,- m: u32,- n: u32,- k: u32,-) -> Option<GemvTile> {- quantized_tile(gpu_core_count, apple_gpu_family, bits, group, m, n, k, true)-}-#[uzu_test]fn quantized_policy_edges() {- assert_eq!(- quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 64, 4, 8192, 4100),- Some(GemvTile::quantized(8, 4, 1, 32, 4))- );- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8, 4096).is_some());- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 1, 8192, 4096).is_some());- assert_eq!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 9, 8192, 4096), None);- assert_eq!(- quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8192, 4096, true),- Some(GemvTile::quantized(8, 4, 1, 32, 2))- );-}--fn quant_shape(- m: u32,- n: u32,- d_transform: GemmDTransform,-) -> MatmulShape {- MatmulShape {- m,- n,- k: 4096,- b_transpose: true,- b_leading_dimension: None,- b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,- b_is_trellis: false,- b_bits: Some(4),- b_group_size: Some(32),- signed_codes: false,- a_full_precision: true,- gathered: false,- params_layout: Some(QuantParamsLayout::OutputGroup),- d_transform,- }+ assert_eq!(quantized_tile(4, 64, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 4)));+ assert!(quantized_tile(4, 32, 4, 8, true).is_some());+ assert!(quantized_tile(4, 32, 1, 8192, true).is_some());+ assert_eq!(quantized_tile(4, 32, 9, 8192, true), None);+ assert_eq!(quantized_tile(4, 32, 6, 8192, true), None);+ assert_eq!(quantized_tile(4, 32, 6, 64, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));+ assert_eq!(quantized_tile(4, 32, 1, 8192, false), None);+ assert_eq!(quantized_tile(4, 32, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));}#[uzu_test]fn block_unaligned_quantized_k_stays_on_gemv() {let shape = MatmulShape {- m: 1,- n: 64,- k: 1152,- b_transpose: true,- b_leading_dimension: None,- b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,- b_is_trellis: false,- b_bits: Some(4),- b_group_size: Some(64),- signed_codes: false,- a_full_precision: true,- gathered: false,params_layout: Some(QuantParamsLayout::OutputGroup),- d_transform: GemmDTransform::empty(),+ ..shape(1, 64, 1152, 4, 64)};assert!(super::super::kernel::GemvSpecialization::select_shape(@@ -150,7 +77,6 @@ fn block_unaligned_quantized_k_stays_on_gemv() {DataType::BF16,DataType::BF16,LARGE_GPU_CORE_COUNT,- MTLGPUFamily::Apple10,).is_some());@@ -158,36 +84,29 @@ fn block_unaligned_quantized_k_stays_on_gemv() {#[uzu_test]fn specialization_preserves_quantized_route() {- let select = |n, transform| {+ let select = |n| {super::super::kernel::GemvSpecialization::select_shape(- &quant_shape(4, n, transform),+ &MatmulShape {+ params_layout: Some(QuantParamsLayout::OutputGroup),+ ..shape(4, n, 4096, 4, 32)+ },GemmBLayout::RowMajor,DataType::BF16,DataType::BF16,DataType::BF16,LARGE_GPU_CORE_COUNT,- MTLGPUFamily::Apple10,)};- let clean = select(8192, GemmDTransform::empty()).expect("quantized specialization");+ let clean = select(8192).expect("quantized specialization");assert!(clean.output_row_tile() > DEFAULT_RESULTS_PER_SIMDGROUP);- assert!(select(8192 + clean.output_row_tile() / 2, GemmDTransform::empty()).is_some());+ assert!(select(8192 + clean.output_row_tile() / 2).is_some());}#[uzu_test]fn gathered_group_major_is_a_gemv_route() {- let mut shape = quant_shape(1, 8192, GemmDTransform::empty());- shape.params_layout = Some(QuantParamsLayout::GroupOutput);+ let mut shape = shape(1, 8192, 4096, 4, 32);shape.gathered = true;assert!(- super::super::kernel::GemvSpecialization::select_tile(- &shape,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- qtile(8, 4),- )- .is_some()+ super::super::kernel::GemvSpecialization::select_tile(&shape, GemmBLayout::RowMajor, qtile(8, 4),).is_some());}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rsdeleted file mode 100644index c9e647499..000000000--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs+++ /dev/null@@ -1,291 +0,0 @@-use metal::MTLGPUFamily;-use uzu_engine_macros::uzu_test;-use xxhash_rust::xxh3::xxh3_64;--use super::*;-use crate::{- backends::{- common::{- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},- kernel::matmul::{MatmulShape, QuantParamsLayout},- },- metal::{- gpu_types::gemm::GemmBLayout,- kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemv::GemvSpecialization},- },- },- data_type::DataType,-};--const FROZEN_PLAN_FINGERPRINT: u64 = 10_794_242_881_707_148_570;--const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [- ("m1", "Apple M1", 8, MTLGPUFamily::Apple7, false),- ("m2", "Apple M2", 10, MTLGPUFamily::Apple8, false),- ("m2-pro", "Apple M2 Pro", 19, MTLGPUFamily::Apple8, false),- ("m3-max", "Apple M3 Max", 40, MTLGPUFamily::Apple9, false),- ("m4", "Apple M4", 10, MTLGPUFamily::Apple9, false),- ("m4-pro", "Apple M4 Pro", 20, MTLGPUFamily::Apple9, false),- ("m5-max", "Apple M5 Max", 40, MTLGPUFamily::Apple10, true),-];-const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 2] = [- ("W4/ZP G64", 4, 64, GemmBPrologueKind::ScaleZeroPointDequant),- ("W8/Symmetric G64", 8, 64, GemmBPrologueKind::ScaleSymmetricDequant),-];-const SHAPES: [(&str, u32, u32); 6] = [- ("down", 5120, 17408),- ("gate", 6144, 5120),- ("gate-up", 34816, 5120),- ("projection-in", 16480, 5120),- ("projection-out", 5120, 6144),- ("readout", 248320, 5120),-];--fn problem(- m: u32,- n: u32,- k: u32,- bits: u32,- group: u32,- prologue: GemmBPrologueKind,-) -> MatmulShape {- MatmulShape {- m,- n,- k,- b_transpose: true,- b_leading_dimension: None,- b_prologue: prologue,- b_is_trellis: false,- b_bits: Some(bits),- b_group_size: Some(group),- signed_codes: false,- a_full_precision: true,- gathered: false,- params_layout: Some(QuantParamsLayout::OutputGroup),- d_transform: GemmDTransform::empty(),- }-}--#[uzu_test]-fn table_is_complete_and_fingerprint_is_stable() {- let mut canonical = Vec::new();- let mut matched_rows = vec![false; ROWS.len()];- for &(device_label, device_name, gpu_core_count, apple_gpu_family, supports_mxu) in &DEVICES {- for &(format_name, bits, group, prologue) in &FORMATS {- for m in 2..=7 {- for &(shape_name, n, k) in SHAPES.iter().filter(|shape| bits == 4 || shape.0 != "gate") {- let mask = shape(n, k);- let matches: Vec<_> = ROWS- .iter()- .enumerate()- .filter(|row| {- row.1.device_name == device_name- && row.1.apple_gpu_family == apple_gpu_family- && row.1.bits == bits- && row.1.group == group- && row.1.m == m- && row.1.shapes & mask != 0- })- .collect();- assert_eq!(matches.len(), 1, "route coverage for {device_label} {format_name} M={m} {shape_name}");- let (row_index, row) = matches[0];- matched_rows[row_index] = true;- let selected = row.route;- canonical.push(format!("{device_label}|{format_name}|{shape_name}|{m}|{n}|{k}|{selected:?}"));- let problem = problem(m, n, k, bits, group, prologue);- assert_eq!(route(device_name, apple_gpu_family, supports_mxu, &problem, true), Some(selected));- let runtime = MatmulMetalKernel::choose_dispatch(- &problem,- GemmBLayout::RowMajor,- device_name,- gpu_core_count,- apple_gpu_family,- supports_mxu,- DataType::BF16,- DataType::BF16,- DataType::BF16,- );- if let QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) = selected {- let specialization = GemvSpecialization::select_tile(- &problem,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- tile,- )- .expect("stored GEMV tile must be legal");- assert!(matches!(runtime, MatmulDispatch::Gemv(actual) if actual == specialization));- } else if let QmvRoute::MainGemm(plan) = selected {- assert!(matches!(runtime, MatmulDispatch::Gemm(actual) if actual == plan));- }- }- }- }- }- assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");- canonical.sort();- assert_eq!(canonical.len(), 462);- assert_eq!(xxh3_64(canonical.join("\n").as_bytes()), FROZEN_PLAN_FINGERPRINT);-}--#[uzu_test]-fn weaver_linears_select_the_tuned_tile() {- for n in [5120, 6144] {- let mut p = problem(4, n, 2048, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- p.params_layout = Some(QuantParamsLayout::GroupOutput);- p.d_transform = GemmDTransform::RHT;- let tile = GemvTile::quantized_output_tile(2, 16, 4, 8, 1);- assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::Tuned(tile)));- assert!(- GemvSpecialization::select_tile(- &p,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- tile- )- .is_some()- );- }-}--#[uzu_test]-fn exact_lookup_rejects_non_matrix_inputs() {- let p = problem(2, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- assert!(route("Apple M1", MTLGPUFamily::Apple7, false, &p, true).is_some());- for mutate in [- |p: &mut MatmulShape| p.m = 1,- |p: &mut MatmulShape| p.n = 1,- |p: &mut MatmulShape| p.b_bits = Some(8),- |p: &mut MatmulShape| p.gathered = true,- ] {- let mut rejected = p;- mutate(&mut rejected);- assert!(route("Apple M1", MTLGPUFamily::Apple7, false, &rejected, true).is_none());- }- assert!(route("Apple M1", MTLGPUFamily::Apple7, false, &p, false).is_none());-- let mut rht = p;- rht.d_transform = GemmDTransform::RHT;- rht.signed_codes = true;- let selected =- route("Apple M1", MTLGPUFamily::Apple7, false, &rht, true).expect("RHT must preserve the exact route");- let QmvRoute::Tuned(tile) = selected else {- panic!("test anchor must use a tuned tile");- };- let deferred = GemvSpecialization::select_tile(- &rht,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- tile,- )- .unwrap();- assert_eq!(deferred.output_row_tile(), 16);- assert!(!deferred.fuses_rht());- rht.n -= 1;- assert!(- GemvSpecialization::select_tile(- &rht,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- tile- )- .is_none()- );-- let p = problem(7, 5120, 17408, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);- assert!(matches!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::MainGemm(_))));- assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, false, &p, true), None);-}--#[uzu_test]-fn normal_routing_handles_inputs_outside_the_frozen_matrix() {- for (m, n, k) in [(1, 5120, 17408), (2, 4096, 5120)] {- let problem = problem(m, n, k, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- assert_eq!(route("Apple M1", MTLGPUFamily::Apple7, false, &problem, true), None);- let specialization = GemvSpecialization::select_shape(- &problem,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- 8,- MTLGPUFamily::Apple7,- )- .expect("normal M1 policy should select GEMV for this anchor");- assert!(matches!(- MatmulMetalKernel::choose_dispatch(- &problem, GemmBLayout::RowMajor,- "Apple M1",- 8,- MTLGPUFamily::Apple7,- false,- DataType::BF16,- DataType::BF16,- DataType::BF16,- ),- MatmulDispatch::Gemv(actual) if actual == specialization- ));- }-- let mut fp = problem(1, 1024, 512, 4, 32, GemmBPrologueKind::ScaleZeroPointDequant);- fp.b_prologue = GemmBPrologueKind::FullPrecision;- fp.b_bits = None;- fp.b_group_size = None;- let mut with_rht = fp;- with_rht.d_transform = GemmDTransform::RHT;- for (cores, family) in [(8, MTLGPUFamily::Apple7), (40, MTLGPUFamily::Apple10)] {- let plain = GemvSpecialization::select_shape(- &fp,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- cores,- family,- )- .expect("generic M=1 FP GEMV tile");- let rht = GemvSpecialization::select_shape(- &with_rht,- GemmBLayout::RowMajor,- DataType::BF16,- DataType::BF16,- DataType::BF16,- cores,- family,- )- .expect("generic M=1 FP RHT GEMV tile");- assert!(plain.output_row_tile() < 32);- assert_eq!(rht, plain);- }-}--#[uzu_test]-fn family_lookup_requires_one_unanimous_route() {- let m1_route = problem(4, 34816, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- let families: [(&MatmulShape, MTLGPUFamily, bool, &str, &[&str]); 4] = [- (&m1_route, MTLGPUFamily::Apple7, false, "Apple M1", &["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"]),- (&unanimous, MTLGPUFamily::Apple8, false, "Apple M2", &["Apple M2 Max", "Apple M2 Ultra"]),- (&unanimous, MTLGPUFamily::Apple9, false, "Apple M3 Max", &["Apple M3", "Apple M3 Pro", "Apple M4 Max"]),- (&unanimous, MTLGPUFamily::Apple10, true, "Apple M5 Max", &["Apple M5", "Apple M5 Pro"]),- ];- for (shape, family, supports_mxu, measured_device, aliases) in families {- let measured = route(measured_device, family, supports_mxu, shape, true);- for device in aliases {- assert_eq!(route(device, family, supports_mxu, shape, true), measured);- }- }-- let disagreement = problem(3, 5120, 6144, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- assert_eq!(route("Apple M2 Max", MTLGPUFamily::Apple8, false, &disagreement, true), None);- assert!(route("Apple M2", MTLGPUFamily::Apple8, false, &disagreement, true).is_some());- assert_eq!(route("Apple A11 GPU", MTLGPUFamily::Apple4, false, &m1_route, true), None);-}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rsindex eb5b606a5..bb707a29f 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs@@ -12,24 +12,32 @@ use crate::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,- gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},+ gpu_types::{+ activation_type::ActivationType, quantization::QuantizationMode,+ quantization_method::QuantizationMethod,+ },kernel::{- Kernels,+ ActivationQuantization, ActivationTransform, GatedActMul, GatedActMulSettings, Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{MatmulDOps, MatmulKernel, MatmulShape, QuantParams, QuantParamsLayout, QuantParamsStrides},+ matmul::{+ ActivationFormat, Int8CodeLayout, MatmulA, MatmulArguments, MatmulKernel, MatmulShape, QuantParams,+ QuantParamsLayout, QuantParamsStrides, packed_half_bytes,+ },},},- cpu::Cpu,metal::{Metal,context::MetalContext,- gpu_types::gemm::GemmBLayout,- kernel::matmul::{MatmulMetalKernel, gemm::GemmEngine},+ gpu_types::gemm::{GemmBLayout, GemmTiling},+ kernel::matmul::{+ GemmKernel, MatmulMetalKernel, MatmulOutputWork,+ gemm::{GemmEngine, GemmPlan},+ },},},- data_type::DataTypeRepr,+ data_type::{DataType, DataTypeRepr},tests::{- helpers::buffer_to_vec,+ helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},matmul::{QuantBuffers, QuantInput, quant::run_quant_cpu, quant_arguments},},};@@ -41,7 +49,15 @@ fn tile_weights<T: DataTypeRepr + Float>(context: &MetalContext,) {let shape = MatmulShape {- a_full_precision: false,+ a: ActivationFormat::Int8(+ ActivationQuantization::new(+ ACTIVATION_SCALE_GROUP_SIZE,+ input.group_size,+ false,+ Int8CodeLayout::for_right_bits(DataType::from(input.mode).size_in_bits() as u32).unwrap(),+ )+ .unwrap(),+ ),..MatmulShape::from_arguments(&quant_arguments(buffers, input))};buffers.weights_layout =@@ -66,16 +82,6 @@ fn optimize_weights_layout_prepares_unsigned_codes() {let signed_shape = MatmulShape::from_arguments(&quant_arguments(&mut expected, &signed));let unsigned_shape = MatmulShape::from_arguments(&quant_arguments(&mut actual, &unsigned));- let unsupported_shape = MatmulShape {- a_full_precision: true,- ..unsigned_shape- };- assert_eq!(- kernel.optimize_weights_layout(&unsupported_shape, &mut actual.w, &context).unwrap(),- (GemmBLayout::RowMajor, false),- );- assert_eq!(buffer_to_vec::<Metal, u32>(&context, &actual.w), unsigned.weights_for_upload());-assert_eq!(kernel.optimize_weights_layout(&signed_shape, &mut expected.w, &context).unwrap(),(GemmBLayout::MxuTiled, true),@@ -94,13 +100,8 @@ fn run_quant_metal<T: DataTypeRepr + Float>(dispatch: Option<GemmEngine>,) -> Vec<T> {let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(- context,- T::data_type(),- T::data_type(),- T::data_type(),- )- .expect("MatmulMetalKernel");+ let mut matmul =+ MatmulMetalKernel::new(context, T::data_type(), T::data_type(), T::data_type()).expect("MatmulMetalKernel");if gemm_b_layout == GemmBLayout::MxuTiled {tile_weights(&matmul, &mut buffers, input, context);}@@ -115,17 +116,6 @@ fn run_quant_metal<T: DataTypeRepr + Float>(buffer_to_vec::<Metal, T>(context, &buffers.y)}-fn check_tolerance(- expected: f32,- actual: f32,- rel_tol: f64,- abs_tol: f64,-) -> bool {- let diff = (expected - actual).abs() as f64;- let tol = abs_tol.max(expected.abs() as f64 * rel_tol);- diff <= tol-}-#[uzu_test]fn quant_params_layout_table() {let cases = [@@ -153,18 +143,13 @@ fn assert_parity<T: DataTypeRepr + Float + Debug + Display>(abs_tol: f64,) {assert_eq!(reference.len(), actual.len(), "{label}: length mismatch");- let mut errors = 0usize;- for (i, (&exp, &got)) in reference.iter().zip(actual.iter()).enumerate() {- let exp_f32 = exp.to_f32().unwrap();- let got_f32 = got.to_f32().unwrap();- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {- if errors < 5 {- eprintln!(" {label} idx={i} expected={exp_f32} got={got_f32} diff={}", (exp_f32 - got_f32).abs());- }- errors += 1;- }+ for (i, (&expected, &actual)) in reference.iter().zip(actual).enumerate() {+ let expected = expected.to_f32().unwrap();+ let actual = actual.to_f32().unwrap();+ let diff = f64::from((expected - actual).abs());+ let tolerance = abs_tol.max(f64::from(expected.abs()) * rel_tol);+ assert!(diff <= tolerance, "{label}[{i}]: expected {expected}, got {actual}, diff {diff} > {tolerance}");}- assert_eq!(errors, 0, "{label}: {errors} mismatches");}fn run_parity<T: DataTypeRepr + Float + Debug + Display>(@@ -242,6 +227,8 @@ fn parity_bf16_8bit_splitk(#[case::m4_gs32_4bit_zp(4, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]#[case::m8_gs32_4bit_zp(8, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]#[case::m8_gs32_4bit_mlx(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]+#[case::m3_w4_g32_zp_k1024(3, 1024, 72, 32, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::m3_w8_g32_sym_k1024(3, 1024, 72, 32, 8, QuantizationMethod::ScaleSymmetric)]fn parity_gemv_bf16(#[case] m: u32,#[case] k: u32,@@ -256,81 +243,24 @@ fn parity_gemv_bf16(#[uzu_test]fn parity_bf16_gs32_4bit_mlx_with_bias() {let context = block_on(MetalContext::new()).expect("Metal context");- let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);+ let mut input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);- let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.5 + 0.1 * (j % 5) as f32).collect();- let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();+ input.output_bias = Some((0..input.n).map(|j| bf16::from_f32(0.5 + 0.1 * (j % 5) as f32)).collect());- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let bias_pp_buf = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &bias_t);- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(- &context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("MatmulMetalKernel");-- let mut command_buffer = context.create_command_buffer(None, None).unwrap();- let mut args = quant_arguments(&mut buffers, &input);- args.output.ops = MatmulDOps {- bias: Some(&bias_pp_buf),- ..MatmulDOps::none()- };- matmul.encode_with_gemm_engine(args, GemmEngine::Simdgroup, &mut command_buffer).expect("encode quant with bias");- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);-- let mut reference = run_quant_cpu::<bf16>(&input);- for i in 0..(input.m as usize) {- for j in 0..(input.n as usize) {- let idx = i * input.n as usize + j;- reference[idx] = bf16::from_f32(reference[idx].to_f32() + bias_f32[j]);- }- }+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, Some(GemmEngine::Simdgroup));+ let reference = run_quant_cpu::<bf16>(&input);assert_parity::<bf16>("with_bias", &reference, &actual, 0.05, 0.4);}#[uzu_test]fn parity_bf16_gemv_qmv_fused_scale_bias() {let context = block_on(MetalContext::new()).expect("Metal context");- let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);-- let scale = 2.0_f32;- let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.25 + 0.1 * (j % 7) as f32).collect();- let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();-- let raw = run_quant_cpu::<bf16>(&input);- let reference: Vec<bf16> = (0..input.m as usize)- .flat_map(|i| {- (0..input.n as usize).map(move |j| {- let idx = i * input.n as usize + j;- (idx, j)- })- })- .map(|(idx, j)| bf16::from_f32(scale * raw[idx].to_f32() + bias_f32[j]))- .collect();+ let mut input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let bias_buf = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &bias_t);- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(- &context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("MatmulMetalKernel");-- let mut command_buffer = context.create_command_buffer(None, None).unwrap();- let mut args = quant_arguments(&mut buffers, &input);- args.output.ops = MatmulDOps {- ab_scale: scale,- bias: Some(&bias_buf),- ..MatmulDOps::none()- };- matmul.encode(args, &mut command_buffer).expect("encode quant gemv with scale+bias");- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);+ input.ab_scale = 2.0;+ input.output_bias = Some((0..input.n).map(|j| bf16::from_f32(0.25 + 0.1 * (j % 7) as f32)).collect());+ let reference = run_quant_cpu::<bf16>(&input);+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, None);assert_parity::<bf16>("gemv_qmv_scale_bias", &reference, &actual, 0.05, 0.4);}@@ -396,66 +326,17 @@ fn parity_bf16_quant_rht(let context = block_on(MetalContext::new()).expect("Metal context");// W4/G64 matches the Qwen short-prefill route; keep the decode anchor at G32.let input = QuantInput::<bf16>::new(m, k, n, group_size, 4, method, 0);- let input = if group_size == 64 {+ let mut input = if group_size == 64 {input.with_group_output()} else {input};- let rht: Vec<i32> = (0..input.n as usize)- .map(|i| {- if i % 2 == 0 {- 1- } else {- -1- }- })- .collect();- let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.3 + 0.05 * (j % 4) as f32).collect();- let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();+ input.output_bias = with_bias.then(|| (0..input.n).map(|j| bf16::from_f32(0.3 + 0.05 * (j % 4) as f32)).collect());+ input.output_rht_factors = Some((0..input.n).map(|i| 1 - 2 * (i % 2) as i32).collect());// CPU reference applies the RHT through the CPU matmul kernel.- let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("Cpu context");- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);- let cpu_rht = crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, &rht);- let cpu_bias = crate::tests::helpers::create_buffer_with_data::<Cpu, bf16>(&cpu_context, &bias_t);- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(- &cpu_context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("MatmulCpuKernel");- let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("cpu command buffer");- let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);- cpu_args.output.ops = MatmulDOps {- bias: with_bias.then_some(&cpu_bias),- rht_factors: Some(&cpu_rht),- ..MatmulDOps::none()- };- cpu_matmul.encode(cpu_args, &mut cpu_command_buffer).expect("cpu encode quant+rht");- block_on(cpu_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let reference = buffer_to_vec::<Cpu, bf16>(&cpu_context, &cpu_buffers.y);-- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let metal_rht = crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, &rht);- let metal_bias = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &bias_t);- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(- &context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("MatmulMetalKernel");- let mut command_buffer = context.create_command_buffer(None, None).unwrap();- let mut args = quant_arguments(&mut buffers, &input);- args.output.ops = MatmulDOps {- bias: with_bias.then_some(&metal_bias),- rht_factors: Some(&metal_rht),- ..MatmulDOps::none()- };- matmul.encode(args, &mut command_buffer).expect("encode quant matmul with rht");- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);+ let reference = run_quant_cpu::<bf16>(&input);+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, None);assert_parity::<bf16>(if with_bias {@@ -667,21 +548,24 @@ fn a8w4_zero_point_tail_parity(#[case] m: u32) {#[rstest]#[test_attr(uzu_test)]-#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256u32)]-#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256u32)]-#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256u32)]+#[case::w4_bias(4, QuantizationMethod::ScaleBias, 256, 2, 32)]+#[case::w4_zp(4, QuantizationMethod::ScaleZeroPoint, 256, 2, 32)]+#[case::w8_sym(8, QuantizationMethod::ScaleSymmetric, 256, 2, 32)]+#[case::direct_w8_m5(8, QuantizationMethod::ScaleSymmetric, 1024, 5, 64)]fn signed_weights_full_precision_activations_parity_bf16(#[case] bits: u32,#[case] method: QuantizationMethod,#[case] k: u32,+ #[case] m: u32,+ #[case] group_size: u32,) {let context = block_on(MetalContext::new()).expect("Metal context");- let (m, n, group_size) = (2u32, 128u32, 32u32);+ let n = 128;let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0).with_signed_weight_codes();let reference_input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0);let reference = run_quant_cpu::<bf16>(&reference_input);- for (label, dispatch) in [("gemv", None), ("simdgroup", Some(GemmEngine::Simdgroup))] {+ for (label, dispatch) in [("auto", None), ("simdgroup", Some(GemmEngine::Simdgroup))] {let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, dispatch);assert_parity::<bf16>(&format!("signed weights FP-A {label} bits={bits} method={method:?}"),@@ -715,9 +599,10 @@ fn a8w_mxu_output_bias_parity_bf16(72u32},);- let (input, reference_input) = QuantInput::<bf16>::new(m, k, n, 32, bits, QuantizationMethod::ScaleSymmetric, 0)- .with_group_output()- .with_prepared_a_and_reference(ACTIVATION_SCALE_GROUP_SIZE, None);+ let (mut input, mut reference_input) =+ QuantInput::<bf16>::new(m, k, n, 32, bits, QuantizationMethod::ScaleSymmetric, 0)+ .with_group_output()+ .with_prepared_a_and_reference(ACTIVATION_SCALE_GROUP_SIZE, None);let output_bias: Vec<bf16> = (0..n).map(|column| bf16::from_f32(0.25 + 0.05 * (column % 7) as f32)).collect();let output_hadamard_factors: Option<Vec<i32>> = with_output_hadamard.then(|| {(0..n)@@ -730,58 +615,14 @@ fn a8w_mxu_output_bias_parity_bf16(}).collect()});+ input.output_bias = Some(output_bias.clone());+ reference_input.output_bias = Some(output_bias);+ input.output_rht_factors = output_hadamard_factors.clone();+ reference_input.output_rht_factors = output_hadamard_factors;- let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("CPU context");- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &reference_input);- let cpu_output_hadamard_factors = output_hadamard_factors- .as_ref()- .map(|factors| crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, factors));- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(- &cpu_context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("CPU matmul kernel");- let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");- let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &reference_input);- cpu_arguments.output.ops = MatmulDOps {- rht_factors: cpu_output_hadamard_factors.as_ref(),- ..MatmulDOps::none()- };- cpu_matmul.encode(cpu_arguments, &mut cpu_command_buffer).expect("CPU A8W matmul");- block_on(cpu_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let mut reference = buffer_to_vec::<Cpu, bf16>(&cpu_context, &cpu_buffers.y);- for row in reference.chunks_exact_mut(n as usize) {- for (value, bias) in row.iter_mut().zip(&output_bias) {- *value = bf16::from_f32(value.to_f32() + bias.to_f32());- }- }+ let reference = run_quant_cpu::<bf16>(&reference_input);- let mut metal_buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let metal_output_bias = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &output_bias);- let metal_output_hadamard_factors = output_hadamard_factors- .as_ref()- .map(|factors| crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, factors));- let mut metal_matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(- &context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("Metal matmul kernel");- let mut metal_command_buffer = context.create_command_buffer(None, None).unwrap();- let mut metal_arguments = quant_arguments(&mut metal_buffers, &input);- metal_arguments.output.ops = MatmulDOps {- bias: Some(&metal_output_bias),- rht_factors: metal_output_hadamard_factors.as_ref(),- ..MatmulDOps::none()- };- metal_matmul- .encode_with_gemm_engine(metal_arguments, GemmEngine::Mxu, &mut metal_command_buffer)- .expect("Metal A8W MXU matmul with output bias");- block_on(metal_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let actual = buffer_to_vec::<Metal, bf16>(&context, &metal_buffers.y);+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, Some(GemmEngine::Mxu));assert_parity::<bf16>(&format!("A8W{bits} MXU output bias, output Hadamard={with_output_hadamard}"),@@ -808,13 +649,14 @@ fn gemv_reads_mxu_tiled_layout_bf16(#[case] bits: u32,#[case] method: QuantizationMethod,#[values(false, true)] output_hadamard: bool,+ #[values(1, 2, 3, 4)] m: u32,) {let context = block_on(MetalContext::new()).expect("Metal context");if !context.supports_mxu {return;}- let input = QuantInput::<bf16>::new(1, k, n, group_size, bits, method, 0).with_group_output();- let input = if bits == 8 {+ let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0).with_group_output();+ let mut input = if bits == 8 {input.with_signed_weight_codes()} else {input@@ -828,49 +670,15 @@ fn gemv_reads_mxu_tiled_layout_bf16(}}).collect();+ input.output_rht_factors = output_hadamard.then_some(factors);- let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("CPU context");- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);- let cpu_factors = crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, &factors);- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(- &cpu_context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("CPU matmul kernel");- let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");- let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &input);- cpu_arguments.output.ops = MatmulDOps {- rht_factors: output_hadamard.then_some(&cpu_factors),- ..MatmulDOps::none()- };- cpu_matmul.encode(cpu_arguments, &mut cpu_command_buffer).expect("CPU matmul");- block_on(cpu_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let reference = buffer_to_vec::<Cpu, bf16>(&cpu_context, &cpu_buffers.y);-- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);- let metal_factors = crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, &factors);- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(- &context,- bf16::data_type(),- bf16::data_type(),- bf16::data_type(),- )- .expect("Metal matmul kernel");- tile_weights(&matmul, &mut buffers, &input, &context);- let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");- let mut arguments = quant_arguments(&mut buffers, &input);- arguments.output.ops = MatmulDOps {- rht_factors: output_hadamard.then_some(&metal_factors),- ..MatmulDOps::none()- };- matmul.encode(arguments, &mut command_buffer).expect("Metal matmul");- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);+ let reference = run_quant_cpu::<bf16>(&input);+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::MxuTiled, None);assert_parity::<bf16>(- &format!("MxuTiled GEMV k={k} n={n} gs={group_size} bits={bits} method={method:?} hadamard={output_hadamard}"),+ &format!(+ "MxuTiled GEMV m={m} k={k} n={n} gs={group_size} bits={bits} method={method:?} hadamard={output_hadamard}"+ ),&reference,&actual,0.05,@@ -928,3 +736,115 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(0.5,);}++fn run_packed_half_pipeline(+ group_size: u32,+ gated: bool,+ (batch, n): (u32, u32),+ with_bias: bool,+ with_rht: bool,+ soft_cap: Option<f32>,+ method: QuantizationMethod,+ split_k: u32,+) {+ let k = 512;+ let source_width = k * (1 + gated as u32);+ let source: Vec<bf16> = (0..batch * source_width).map(|i| bf16::from_f32((i as f32 * 0.013).sin() * 0.2)).collect();+ let signs = |len: u32| (0..len).map(|i| 1 - 2 * (i % 3 == 0) as i32).collect::<Vec<_>>();+ let factors = signs(k);+ let format = ActivationFormat::PackedHalf {+ group_size,+ };+ let mut input = QuantInput::<bf16>::new(batch, k, n, group_size, 4, method, 19).with_group_output();+ input.ab_scale = 0.75;+ input.output_bias = with_bias.then(|| (0..n).map(|i| bf16::from_f32(1.0 + 0.5 * (i % 3) as f32)).collect());+ input.output_rht_factors = with_rht.then(|| signs(n));+ input.soft_cap = soft_cap;+ let context = block_on(MetalContext::new()).expect("Metal context");+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);+ let source_buffer = create_buffer_with_data::<Metal, bf16>(&context, &source);+ let factors_buffer = create_buffer_with_data::<Metal, i32>(&context, &factors);+ let mut reference_a = create_buffer::<Metal, bf16>(&context, (batch * k) as usize);+ let mut gemm = GemmKernel::new(DataType::BF16, DataType::BF16, DataType::BF16);+ let output_work = MatmulOutputWork::new(&context, DataType::BF16, DataType::BF16).unwrap();+ let mut packed = create_buffer::<Metal, u32>(&context, packed_half_bytes(k, group_size) / size_of::<u32>());+ let mut command_buffer = context.create_command_buffer(None, None).expect("Metal command buffer");+ if !gated {+ ActivationTransform::<Metal>::input_rht(&context, DataType::BF16, false).expect("RHT reference").encode_fp(+ &source_buffer,+ &mut reference_a,+ &factors_buffer,+ batch,+ k,+ &mut command_buffer,+ );+ ActivationTransform::<Metal>::quantize(&context, DataType::BF16, format)+ .expect("Metal RHT PackedHalf pack")+ .encode_pack_half(&source_buffer, &mut packed, &factors_buffer, batch, k, &mut command_buffer);+ } else {+ GatedActMul::<Metal>::full_precision(&context, DataType::BF16, true, true, GatedActMulSettings::default())+ .expect("gate reference")+ .encode_fp(+ &source_buffer,+ None::<&<Metal as Backend>::GlobalGpuBuffer>,+ &mut reference_a,+ Some(&factors_buffer),+ k,+ batch,+ 0,+ 0,+ ActivationType::SILU,+ &mut command_buffer,+ );+ GatedActMul::<Metal>::quantized(&context, DataType::BF16, format, GatedActMulSettings::default())+ .expect("Metal PackedHalf gated activation")+ .encode_pack_half(+ &source_buffer,+ &mut packed,+ &factors_buffer,+ k,+ batch,+ ActivationType::SILU,+ &mut command_buffer,+ );+ }+ let arguments = quant_arguments(&mut buffers, &input);+ let arguments = MatmulArguments {+ a: MatmulA::PackedHalf {+ values: &packed,+ group_size,+ },+ b: arguments.b,+ b_leading_dimension: arguments.b_leading_dimension,+ b_transpose: arguments.b_transpose,+ output: arguments.output,+ gather_indices: arguments.gather_indices,+ m: arguments.m,+ n: arguments.n,+ k: arguments.k,+ };+ let plan = GemmPlan {+ engine: GemmEngine::Simdgroup,+ tiling: GemmTiling::Tile8x32x64_Simdgroups1x2x4,+ split_k,+ };+ gemm.encode_plan(arguments, plan, &output_work, &mut command_buffer).expect("prepared W4 GEMM");+ block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();+ input.x = buffer_to_vec::<Metal, bf16>(&context, &reference_a);+ let reference = run_quant_cpu::<bf16>(&input);+ let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);+ assert_parity::<bf16>(&format!("gated={gated} G{group_size} M{batch} K{k}"), &reference, &actual, 0.05, 0.4);+}++#[uzu_test]+fn packed_half_producers_match_bf16_reference() {+ // Fused RHT rounds directly to half; the standalone reference stores BF16 first.+ for (group, gated, n, bias, rht, cap, method, splits) in [+ (32, false, (4, 72), false, false, None, ScaleZeroPoint, 2),+ (64, false, (5, 64), true, false, None, ScaleBias, 1),+ (32, true, (8, 64), true, true, None, ScaleZeroPoint, 1),+ (64, true, (5, 64), true, false, Some(0.5), ScaleBias, 2),+ ] {+ run_packed_half_pipeline(group, gated, n, bias, rht, cap, method, splits);+ }+}diff --git a/crates/uzu-engine/unit/common/matmul/mod.rs b/crates/uzu-engine/unit/common/matmul/mod.rsindex 7d844fd12..1112d03e4 100644--- a/crates/uzu-engine/unit/common/matmul/mod.rs+++ b/crates/uzu-engine/unit/common/matmul/mod.rs@@ -7,6 +7,6 @@ pub use bench::{iter_encode_loop, iter_encode_loop_named};pub use harness::{Case, cpu_reference, deterministic_input};pub use quant::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu, transpose_metadata};pub use shape::{- Shape, all_correctness_shapes, bench_fp_gemm_shapes, bench_quant_gemm_shapes, bench_quant_gemv_shapes,- qwen3_layer_shapes,+ Shape, a8_format, all_correctness_shapes, bench_fp_gemm_shapes, bench_quant_gemm_shapes, bench_quant_gemv_shapes,+ matmul_shape, quantized_matmul_shape, qwen3_layer_shapes,};diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex 6b921832e..88984a91e 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -10,7 +10,7 @@ use crate::{Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},kernel::{- ActivationQuantization, ActivationTransform, Kernels,+ ActivationFormat, ActivationQuantization, ActivationTransform, Kernels,matmul::{Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout,@@ -38,6 +38,10 @@ pub struct QuantInput<T: DataTypeRepr + Float> {pub zero_points: Option<Vec<u8>>,pub biases: Option<Vec<T>>,pub x: Vec<T>,+ pub ab_scale: f32,+ pub soft_cap: Option<f32>,+ pub output_bias: Option<Vec<T>>,+ pub output_rht_factors: Option<Vec<i32>>,pub k: u32,pub n: u32,pub m: u32,@@ -165,6 +169,10 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {zero_points,biases,x,+ ab_scale: 1.0,+ soft_cap: None,+ output_bias: None,+ output_rht_factors: None,k,n,m,@@ -241,8 +249,9 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {code_layout,).expect("supported activation quantization");- let transform = ActivationTransform::<Cpu>::quantize(&context, T::data_type(), quantization)- .expect("CPU activation quantization transform");+ let transform =+ ActivationTransform::<Cpu>::quantize(&context, T::data_type(), ActivationFormat::Int8(quantization))+ .expect("CPU activation quantization transform");let mut command_buffer = context.create_command_buffer(None, None).expect("CPU command buffer");transform.encode_quantize(&input,@@ -292,6 +301,8 @@ pub struct QuantBuffers<B: Backend, T: DataTypeRepr + Float> {pub prepared_a: Option<B::GlobalGpuBuffer>,pub prepared_a_scales: Option<B::GlobalGpuBuffer>,pub prepared_a_group_sums: Option<B::GlobalGpuBuffer>,+ pub output_bias: Option<B::GlobalGpuBuffer>,+ pub output_rht_factors: Option<B::GlobalGpuBuffer>,pub y: B::GlobalGpuBuffer,pub weights_layout: WeightsLayout<B>,_t: std::marker::PhantomData<T>,@@ -339,6 +350,11 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {.as_ref().filter(|prepared| !prepared.group_sums.is_empty()).map(|prepared| create_buffer_with_data::<B, i32>(context, &prepared.group_sums)),+ output_bias: input.output_bias.as_ref().map(|bias| create_buffer_with_data::<B, T>(context, bias)),+ output_rht_factors: input+ .output_rht_factors+ .as_ref()+ .map(|factors| create_buffer_with_data::<B, i32>(context, factors)),y: create_buffer::<B, T>(context, (input.m as usize) * (input.n as usize)),weights_layout: Default::default(),_t: std::marker::PhantomData,@@ -408,6 +424,8 @@ pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(prepared_a,prepared_a_scales,prepared_a_group_sums,+ output_bias,+ output_rht_factors,y,weights_layout,..@@ -420,20 +438,24 @@ pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(// Symmetric weights carry no correction term, so the GEMM never reads these.group_sums: (input.quant_method != QuantizationMethod::ScaleSymmetric).then(|| prepared_a_group_sums.as_ref().expect("prepared activation row sums")),- scale_group_size: prepared.quantization.scale_group_size(),- code_layout: prepared.quantization.code_layout(),- },- None => MatmulA::FullPrecision {- values: &*x,- offset: 0,+ quantization: prepared.quantization,},+ None => MatmulA::full_precision(&*x),};MatmulArguments {a,b,b_leading_dimension: None,b_transpose: true,- output: MatmulOutput::new(y, MatmulDOps::none()),+ output: MatmulOutput::new(+ y,+ MatmulDOps {+ ab_scale: input.ab_scale,+ bias: output_bias.as_ref(),+ rht_factors: output_rht_factors.as_ref(),+ soft_cap: input.soft_cap,+ },+ ),gather_indices: None,m: input.m,n: input.n,diff --git a/crates/uzu-engine/unit/common/matmul/shape.rs b/crates/uzu-engine/unit/common/matmul/shape.rsindex 9ee092cf1..970b25329 100644--- a/crates/uzu-engine/unit/common/matmul/shape.rs+++ b/crates/uzu-engine/unit/common/matmul/shape.rs@@ -1,5 +1,13 @@use derive_more::Display;+use crate::backends::common::{+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+ kernel::{+ ActivationQuantization,+ matmul::{ActivationFormat, Int8CodeLayout, MatmulShape, QuantParamsLayout},+ },+};+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]#[display("M[{m}]K[{k}]N[{n}]")]pub struct Shape {@@ -111,3 +119,51 @@ pub fn qwen3_layer_shapes(bits: u32) -> impl Iterator<Item = (&'static str, Shap.filter(move |&&(_, k, _)| k.is_multiple_of(block_size)).flat_map(move |&(label, k, n)| ms.iter().map(move |&m| (label, Shape::new(m, k, n))))}++pub fn matmul_shape(+ m: u32,+ n: u32,+ k: u32,+) -> MatmulShape {+ MatmulShape {+ m,+ n,+ k,+ b_transpose: true,+ b_leading_dimension: None,+ b_prologue: GemmBPrologueKind::FullPrecision,+ b_is_trellis: false,+ b_bits: None,+ b_group_size: None,+ signed_codes: false,+ a: ActivationFormat::Bf16,+ gathered: false,+ params_layout: Some(QuantParamsLayout::OutputGroup),+ d_transform: GemmDTransform::empty(),+ }+}++pub fn quantized_matmul_shape(+ m: u32,+ n: u32,+ k: u32,+ bits: u32,+ group: u32,+) -> MatmulShape {+ MatmulShape {+ b_prologue: if bits == 8 {+ GemmBPrologueKind::ScaleSymmetricDequant+ } else {+ GemmBPrologueKind::ScaleZeroPointDequant+ },+ b_bits: Some(bits),+ b_group_size: Some(group),+ signed_codes: bits == 8,+ params_layout: Some(QuantParamsLayout::GroupOutput),+ ..matmul_shape(m, n, k)+ }+}++pub fn a8_format() -> ActivationFormat {+ ActivationFormat::Int8(ActivationQuantization::new(128, 128, false, Int8CodeLayout::GroupedByNibble).unwrap())+}