Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Optimize and unify low-batch matmul (#1023)

CCYehece61cb866bf
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rs b/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rs
index 0e28f0315..0f086ae56 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/activation_transform.rs
@@ -5,4 +5,5 @@ pub enum ActivationTransformOp {
OutputRht,
Quantize,
QuantizeWithGroupSums,
+ PackHalf,
}
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rs
index d4605fbdb..300310824 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/gated_act_mul.rs
@@ -4,4 +4,5 @@ pub enum GatedActMulOp {
FullPrecision,
Quantize,
QuantizeWithGroupSums,
+ PackHalf,
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
index 73018d002..d5ff4cc24 100644
--- a/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/activation_transform.rs
@@ -2,7 +2,10 @@ use crate::{
backends::common::{
Backend, BufferMut, BufferRef, CommandBuffer, Kernels,
gpu_types::{activation_transform::ActivationTransformOp, hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE},
- kernel::{ActivationTransformKernel, matmul::Int8CodeLayout},
+ kernel::{
+ ActivationTransformKernel,
+ matmul::{ActivationFormat, Int8CodeLayout},
+ },
},
data_type::DataType,
};
@@ -16,7 +19,7 @@ fn assert_row_width(element_count: u32) {
pub const ACTIVATION_SCALE_GROUP_SIZE: u32 = 128;
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ActivationQuantization {
scale_group_size: u32,
sum_group_size: Option<u32>,
@@ -55,7 +58,7 @@ pub struct ActivationTransform<B: Backend> {
kernel: <B::Kernels as Kernels>::ActivationTransformKernel,
in_place: bool,
has_bias: bool,
- quantization: Option<ActivationQuantization>,
+ format: ActivationFormat,
}
impl<B: Backend> ActivationTransform<B> {
@@ -65,12 +68,21 @@ impl<B: Backend> ActivationTransform<B> {
bias_data_type: Option<DataType>,
ops: ActivationTransformOp,
in_place: bool,
- quantization: Option<ActivationQuantization>,
+ format: ActivationFormat,
) -> Result<Self, B::Error> {
let has_bias = bias_data_type.is_some();
- let (codes_grouped_by_nibble, scale_group_size, sum_group_size) = match quantization {
- Some(q) => (q.code_layout.is_grouped_by_nibble(), q.scale_group_size, q.sum_group_size),
- None => (false, HADAMARD_TRANSFORM_BLOCK_SIZE, None),
+ let (codes_grouped_by_nibble, scale_group_size, sum_group_size) = match format {
+ ActivationFormat::Int8(q) => (
+ q.code_layout().is_grouped_by_nibble(),
+ q.scale_group_size(),
+ q.sum_group_size().unwrap_or(q.scale_group_size()),
+ ),
+ ActivationFormat::PackedHalf {
+ group_size,
+ } => (false, HADAMARD_TRANSFORM_BLOCK_SIZE, group_size),
+ ActivationFormat::Bf16 | ActivationFormat::Trellis => {
+ (false, HADAMARD_TRANSFORM_BLOCK_SIZE, HADAMARD_TRANSFORM_BLOCK_SIZE)
+ },
};
let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(
context,
@@ -80,14 +92,14 @@ impl<B: Backend> ActivationTransform<B> {
codes_grouped_by_nibble,
in_place,
scale_group_size,
- sum_group_size.unwrap_or(HADAMARD_TRANSFORM_BLOCK_SIZE),
+ sum_group_size,
has_bias,
)?;
Ok(Self {
kernel,
in_place,
has_bias,
- quantization,
+ format,
})
}
@@ -96,7 +108,7 @@ impl<B: Backend> ActivationTransform<B> {
data_type: DataType,
in_place: bool,
) -> Result<Self, B::Error> {
- Self::new(context, data_type, None, ActivationTransformOp::InputRht, in_place, None)
+ Self::new(context, data_type, None, ActivationTransformOp::InputRht, in_place, ActivationFormat::Bf16)
}
pub fn output_rht(
@@ -105,30 +117,34 @@ impl<B: Backend> ActivationTransform<B> {
bias_data_type: Option<DataType>,
in_place: bool,
) -> Result<Self, B::Error> {
- Self::new(context, data_type, bias_data_type, ActivationTransformOp::OutputRht, in_place, None)
+ Self::new(
+ context,
+ data_type,
+ bias_data_type,
+ ActivationTransformOp::OutputRht,
+ in_place,
+ ActivationFormat::Bf16,
+ )
}
pub fn quantize(
context: &B::Context,
data_type: DataType,
- quantization: ActivationQuantization,
+ quantization: ActivationFormat,
) -> Result<Self, B::Error> {
- let op = quantization
- .sum_group_size()
- .map_or(ActivationTransformOp::Quantize, |_| ActivationTransformOp::QuantizeWithGroupSums);
- Self::new(context, data_type, None, op, false, Some(quantization))
- }
-
- pub fn scale_group_size(&self) -> u32 {
- self.quantization.expect("quantized activation transform required").scale_group_size()
- }
-
- pub fn sum_group_size(&self) -> Option<u32> {
- self.quantization.expect("quantized activation transform required").sum_group_size()
+ let op = match quantization {
+ ActivationFormat::Int8(q) if q.sum_group_size().is_some() => ActivationTransformOp::QuantizeWithGroupSums,
+ ActivationFormat::Int8(_) => ActivationTransformOp::Quantize,
+ ActivationFormat::PackedHalf {
+ ..
+ } => ActivationTransformOp::PackHalf,
+ ActivationFormat::Bf16 | ActivationFormat::Trellis => unreachable!(),
+ };
+ Self::new(context, data_type, None, op, false, quantization)
}
- pub fn code_layout(&self) -> Int8CodeLayout {
- self.quantization.expect("quantized activation transform required").code_layout()
+ pub fn format(&self) -> ActivationFormat {
+ self.format
}
/// `input` and `output` must be distinct buffers.
@@ -141,7 +157,7 @@ impl<B: Backend> ActivationTransform<B> {
element_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
- assert!(self.quantization.is_none() && !self.in_place && !self.has_bias);
+ assert!(self.format == ActivationFormat::Bf16 && !self.in_place && !self.has_bias);
assert_row_width(element_count);
self.kernel.encode(
Some(input),
@@ -153,6 +169,7 @@ impl<B: Backend> ActivationTransform<B> {
rht_factors,
batch_size,
element_count,
+ None::<&mut B::ScratchBuffer>,
command_buffer,
);
}
@@ -166,7 +183,7 @@ impl<B: Backend> ActivationTransform<B> {
element_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
- assert!(self.quantization.is_none() && self.in_place && self.has_bias == bias.is_some());
+ assert!(self.format == ActivationFormat::Bf16 && self.in_place && self.has_bias == bias.is_some());
assert_row_width(element_count);
self.kernel.encode(
None::<&B::ScratchBuffer>,
@@ -178,6 +195,7 @@ impl<B: Backend> ActivationTransform<B> {
rht_factors,
batch_size,
element_count,
+ None::<&mut B::ScratchBuffer>,
command_buffer,
);
}
@@ -193,7 +211,9 @@ impl<B: Backend> ActivationTransform<B> {
element_count: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
- let quantization = self.quantization.expect("quantized activation transform required");
+ let ActivationFormat::Int8(quantization) = self.format else {
+ panic!("quantized activation transform required")
+ };
assert_row_width(element_count);
assert!(
element_count.is_multiple_of(quantization.scale_group_size()),
@@ -213,6 +233,33 @@ impl<B: Backend> ActivationTransform<B> {
rht_factors,
batch_size,
element_count,
+ None::<&mut B::ScratchBuffer>,
+ command_buffer,
+ );
+ }
+
+ pub fn encode_pack_half(
+ &self,
+ input: impl BufferRef<Backend = B>,
+ output: impl BufferMut<Backend = B>,
+ rht_factors: impl BufferRef<Backend = B>,
+ batch_size: u32,
+ element_count: u32,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) {
+ assert!(matches!(self.format, ActivationFormat::PackedHalf { .. }) && batch_size <= 8);
+ assert_row_width(element_count);
+ self.kernel.encode(
+ Some(input),
+ None::<&mut B::ScratchBuffer>,
+ None::<&B::GlobalGpuBuffer>,
+ None::<&mut B::ScratchBuffer>,
+ None::<&mut B::ScratchBuffer>,
+ None::<&mut B::ScratchBuffer>,
+ rht_factors,
+ batch_size,
+ element_count,
+ Some(output),
command_buffer,
);
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
index 77dc2bd4d..9a4a6610a 100644
--- a/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/gated_act_mul.rs
@@ -7,7 +7,7 @@ use crate::{
activation_type::ActivationType, gated_act_mul::GatedActMulOp,
hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
},
- kernel::{ActivationQuantization, GatedActMulKernel},
+ kernel::{GatedActMulKernel, matmul::ActivationFormat},
},
config::clipping::ClippingBounds,
data_type::DataType,
@@ -33,7 +33,7 @@ pub struct GatedActMul<B: Backend> {
kernel: <B::Kernels as Kernels>::GatedActMulKernel,
options: GatedActMulOptions,
settings: GatedActMulSettings,
- quantization: Option<ActivationQuantization>,
+ format: ActivationFormat,
}
impl<B: Backend> GatedActMul<B> {
@@ -47,17 +47,17 @@ impl<B: Backend> GatedActMul<B> {
let mut options = GatedActMulOptions::empty();
options.set(GatedActMulOptions::INTERLEAVED, interleaved);
options.set(GatedActMulOptions::HADAMARD, use_hadamard);
- Self::new(context, data_type, options, settings, None)
+ Self::new(context, data_type, options, settings, ActivationFormat::Bf16)
}
pub fn quantized(
context: &B::Context,
data_type: DataType,
- quantization: ActivationQuantization,
+ quantization: ActivationFormat,
settings: GatedActMulSettings,
) -> Result<Self, B::Error> {
let options = GatedActMulOptions::INTERLEAVED | GatedActMulOptions::HADAMARD;
- Self::new(context, data_type, options, settings, Some(quantization))
+ Self::new(context, data_type, options, settings, quantization)
}
fn new(
@@ -65,10 +65,10 @@ impl<B: Backend> GatedActMul<B> {
data_type: DataType,
options: GatedActMulOptions,
settings: GatedActMulSettings,
- quantization: Option<ActivationQuantization>,
+ format: ActivationFormat,
) -> Result<Self, B::Error> {
- let (ops, codes_grouped_by_nibble, scale_group_size, sum_group_size) = match quantization {
- Some(quantization) => (
+ let (ops, codes_grouped_by_nibble, scale_group_size, sum_group_size) = match format {
+ ActivationFormat::Int8(quantization) => (
if quantization.sum_group_size().is_some() {
GatedActMulOp::QuantizeWithGroupSums
} else {
@@ -78,7 +78,12 @@ impl<B: Backend> GatedActMul<B> {
quantization.scale_group_size(),
quantization.sum_group_size().unwrap_or(quantization.scale_group_size()),
),
- None => (GatedActMulOp::FullPrecision, false, HADAMARD_TRANSFORM_BLOCK_SIZE, HADAMARD_TRANSFORM_BLOCK_SIZE),
+ ActivationFormat::PackedHalf {
+ group_size,
+ } => (GatedActMulOp::PackHalf, false, HADAMARD_TRANSFORM_BLOCK_SIZE, group_size),
+ ActivationFormat::Bf16 | ActivationFormat::Trellis => {
+ (GatedActMulOp::FullPrecision, false, HADAMARD_TRANSFORM_BLOCK_SIZE, HADAMARD_TRANSFORM_BLOCK_SIZE)
+ },
};
let kernel = <B::Kernels as Kernels>::GatedActMulKernel::new(
context,
@@ -97,10 +102,14 @@ impl<B: Backend> GatedActMul<B> {
kernel,
options,
settings,
- quantization,
+ format,
})
}
+ pub fn format(&self) -> ActivationFormat {
+ self.format
+ }
+
pub fn encode_fp(
&self,
act_operand: impl BufferRef<Backend = B>,
@@ -114,7 +123,7 @@ impl<B: Backend> GatedActMul<B> {
act_type: ActivationType,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
- assert!(self.quantization.is_none());
+ assert_eq!(self.format, ActivationFormat::Bf16);
assert_eq!(self.options.contains(GatedActMulOptions::INTERLEAVED), value_operand.is_none());
assert_eq!(self.options.contains(GatedActMulOptions::HADAMARD), hadamard_factors.is_some());
assert!(
@@ -141,6 +150,7 @@ impl<B: Backend> GatedActMul<B> {
gate_clip_max,
value_clip_min,
value_clip_max,
+ None::<&mut B::ScratchBuffer>,
command_buffer,
);
}
@@ -157,7 +167,9 @@ impl<B: Backend> GatedActMul<B> {
act_type: ActivationType,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
- let quantization = self.quantization.expect("quantized gated activation required");
+ let ActivationFormat::Int8(quantization) = self.format else {
+ panic!("quantized gated activation required")
+ };
assert!(self.options.contains(GatedActMulOptions::INTERLEAVED));
assert!(self.options.contains(GatedActMulOptions::HADAMARD));
assert!(gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));
@@ -186,6 +198,44 @@ impl<B: Backend> GatedActMul<B> {
gate_clip_max,
value_clip_min,
value_clip_max,
+ None::<&mut B::ScratchBuffer>,
+ command_buffer,
+ );
+ }
+
+ pub fn encode_pack_half(
+ &self,
+ act_operand: impl BufferRef<Backend = B>,
+ output: impl BufferMut<Backend = B>,
+ hadamard_factors: impl BufferRef<Backend = B>,
+ gated_dim: u32,
+ batch_dim: u32,
+ act_type: ActivationType,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) {
+ assert!(matches!(self.format, ActivationFormat::PackedHalf { .. }) && batch_dim <= 8);
+ assert!(gated_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));
+ let (gate_clip_min, gate_clip_max) = self.settings.gate_clipping.into_pair().unzip();
+ let (value_clip_min, value_clip_max) = self.settings.value_clipping.into_pair().unzip();
+ self.kernel.encode(
+ act_operand,
+ None::<&B::ScratchBuffer>,
+ None::<&mut B::ScratchBuffer>,
+ None::<&mut B::ScratchBuffer>,
+ None::<&mut B::ScratchBuffer>,
+ None::<&mut B::ScratchBuffer>,
+ Some(hadamard_factors),
+ gated_dim,
+ batch_dim,
+ 0,
+ 0,
+ act_type,
+ self.settings.activation_alpha,
+ gate_clip_min,
+ gate_clip_max,
+ value_clip_min,
+ value_clip_max,
+ Some(output),
command_buffer,
);
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
index 717da5b7c..030cd9494 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/kernel.rs
@@ -47,10 +47,20 @@ pub trait MatmulKernel: Sized + Send + Sync {
None
}
+ fn activation_preparation(
+ &self,
+ _shape: &MatmulShape,
+ _weights_layout: Self::WeightsLayout,
+ _context: &<Self::Backend as Backend>::Context,
+ ) -> ActivationFormat {
+ ActivationFormat::Bf16
+ }
+
fn select_activation_format(
&self,
_bf16_shape: &MatmulShape,
_weights_layout: Self::WeightsLayout,
+ _prepared: ActivationFormat,
_context: &<Self::Backend as Backend>::Context,
) -> ActivationFormat {
ActivationFormat::Bf16
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
index 4b3473809..dc69c1203 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
@@ -1,8 +1,8 @@
use std::mem::size_of;
-use crate::backends::common::BufferRef;
+use crate::backends::common::kernel::ActivationQuantization;
-#[derive(Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Int8CodeLayout {
Sequential,
GroupedByNibble,
@@ -46,17 +46,21 @@ impl Int8CodeLayout {
}
}
-pub enum MatmulA<T: BufferRef> {
+#[derive(Clone, Copy)]
+pub enum MatmulA<T> {
FullPrecision {
values: T,
offset: usize,
},
+ PackedHalf {
+ values: T,
+ group_size: u32,
+ },
Int8Symmetric {
values: T,
scales: T,
group_sums: Option<T>,
- scale_group_size: u32,
- code_layout: Int8CodeLayout,
+ quantization: ActivationQuantization,
},
Trellis {
values: T,
@@ -64,3 +68,60 @@ pub enum MatmulA<T: BufferRef> {
scales: T,
},
}
+
+impl<T> MatmulA<T> {
+ pub fn full_precision(values: T) -> Self {
+ Self::FullPrecision {
+ values,
+ offset: 0,
+ }
+ }
+
+ pub fn as_ref(&self) -> MatmulA<&T> {
+ match self {
+ Self::FullPrecision {
+ values,
+ offset,
+ } => MatmulA::FullPrecision {
+ values,
+ offset: *offset,
+ },
+ Self::PackedHalf {
+ values,
+ group_size,
+ } => MatmulA::PackedHalf {
+ values,
+ group_size: *group_size,
+ },
+ Self::Int8Symmetric {
+ values,
+ scales,
+ group_sums,
+ quantization,
+ } => MatmulA::Int8Symmetric {
+ values,
+ scales,
+ group_sums: group_sums.as_ref(),
+ quantization: *quantization,
+ },
+ Self::Trellis {
+ values,
+ column_group_sums,
+ scales,
+ } => MatmulA::Trellis {
+ values,
+ column_group_sums,
+ scales,
+ },
+ }
+ }
+}
+
+/// Matches `PackedHalfLayout` in `packed_half_layout.h`.
+pub fn packed_half_bytes(
+ k: u32,
+ group_size: u32,
+) -> usize {
+ assert!(matches!(group_size, 32 | 64) && k.is_multiple_of(group_size));
+ (k / group_size) as usize * (32 * (group_size as usize / 8) + 16) * size_of::<u32>()
+}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
index d70ba1475..1d2030d4a 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
@@ -9,7 +9,7 @@ pub mod routing;
pub use arguments::MatmulArguments;
pub use d_ops::MatmulDOps;
pub use kernel::MatmulKernel;
-pub use matmul_a::{Int8CodeLayout, MatmulA};
+pub use matmul_a::{Int8CodeLayout, MatmulA, packed_half_bytes};
pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};
#[cfg(backend = "metal")]
pub use matmul_b::{TRELLIS_CODES_TAIL_BYTES, TrellisFormat};
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs
index 98d2df91d..83518b514 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs
@@ -2,12 +2,17 @@ use super::{MatmulA, MatmulArguments, QuantParamsLayout};
use crate::backends::common::{
Backend, BufferMut, BufferRef,
gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ kernel::ActivationQuantization,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ActivationFormat {
Bf16,
- Int8,
+ Int8(ActivationQuantization),
+ PackedHalf {
+ group_size: u32,
+ },
+ Trellis,
}
#[derive(Clone, Copy)]
@@ -22,7 +27,7 @@ pub struct MatmulShape {
pub b_bits: Option<u32>,
pub b_group_size: Option<u32>,
pub signed_codes: bool,
- pub a_full_precision: bool,
+ pub a: ActivationFormat,
pub gathered: bool,
pub params_layout: Option<QuantParamsLayout>,
pub d_transform: GemmDTransform,
@@ -50,7 +55,24 @@ impl MatmulShape {
b_bits: arguments.b.bits_per_b(),
b_group_size: arguments.b.group_size(),
signed_codes: arguments.b.signed_codes(),
- a_full_precision: matches!(arguments.a, MatmulA::FullPrecision { .. }),
+ a: match &arguments.a {
+ MatmulA::FullPrecision {
+ ..
+ } => ActivationFormat::Bf16,
+ MatmulA::PackedHalf {
+ group_size,
+ ..
+ } => ActivationFormat::PackedHalf {
+ group_size: *group_size,
+ },
+ MatmulA::Int8Symmetric {
+ quantization,
+ ..
+ } => ActivationFormat::Int8(*quantization),
+ MatmulA::Trellis {
+ ..
+ } => ActivationFormat::Trellis,
+ },
gathered: arguments.gather_indices.is_some(),
params_layout: arguments.b.quant_params_layout(),
d_transform: arguments.output.ops.mask(),
@@ -60,4 +82,8 @@ impl MatmulShape {
pub fn is_quant(&self) -> bool {
self.b_prologue != GemmBPrologueKind::FullPrecision
}
+
+ pub fn a_is_quantized(&self) -> bool {
+ matches!(self.a, ActivationFormat::Int8(_) | ActivationFormat::Trellis)
+ }
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rs
index 034b6672c..5a3a3efaf 100644
--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs
@@ -19,6 +19,7 @@ pub use attention::{
AttentionArguments, AttentionInput, AttentionKernel, AttentionKernelConfig, FullPrecisionInput, Int8Input, KVFormat,
};
pub use gated_act_mul::{GatedActMul, GatedActMulSettings};
+pub use matmul::ActivationFormat;
#[cfg(any(backend = "cpu", test))]
pub use crate::backends::common::kernel::trellis::mixing_dimension;
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
index 2a7e1bb54..a501ae16b 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/activation_transform.rs
@@ -64,6 +64,7 @@ pub fn activation_transform<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Float
rht_factors: *const i32,
batch_size: u32,
element_count: u32,
+ #[optional(ops == ActivationTransformOp::PackHalf)] packed_half_out: Option<*mut u32>,
#[specialize] ops: ActivationTransformOp,
#[specialize] grouped_by_weight_nibble: bool,
#[specialize] in_place: bool,
@@ -79,6 +80,7 @@ pub fn activation_transform<T: DataTypeRepr + Float, BiasT: DataTypeRepr + Float
let columns = element_count as usize;
let input_rht = ops != ActivationTransformOp::OutputRht;
let quantize = matches!(ops, ActivationTransformOp::Quantize | ActivationTransformOp::QuantizeWithGroupSums);
+ assert!(packed_half_out.is_none(), "CPU packed-half activation is not selected");
assert_eq!(bias.is_some(), has_bias);
let mut transformed = vec![0.0f32; columns];
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
index 95256b52c..15f41254a 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/gated_act_mul/gated_act_mul.rs
@@ -41,6 +41,7 @@ pub fn gated_act_mul<T: DataTypeRepr + Float>(
#[optional(clip_gate)] gate_clip_max: Option<f32>,
#[optional(clip_value)] value_clip_min: Option<f32>,
#[optional(clip_value)] value_clip_max: Option<f32>,
+ #[optional(ops == GatedActMulOp::PackHalf)] packed_half_out: Option<*mut u32>,
#[specialize] ops: GatedActMulOp,
#[specialize] grouped_by_weight_nibble: bool,
#[specialize] interleaved: bool,
@@ -54,6 +55,7 @@ pub fn gated_act_mul<T: DataTypeRepr + Float>(
assert_eq!(hadamard_factors.is_some(), use_hadamard);
let quantize = matches!(ops, GatedActMulOp::Quantize | GatedActMulOp::QuantizeWithGroupSums);
let emit_group_sums = ops == GatedActMulOp::QuantizeWithGroupSums;
+ assert!(packed_half_out.is_none(), "CPU packed-half gate is not selected");
assert!(!quantize || use_hadamard, "quantized gate activation requires RHT");
assert!(!quantize || gated_dim.is_multiple_of(activation_scale_group_size));
assert!(!emit_group_sums || gated_dim.is_multiple_of(sum_group_size));
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index 94debb345..7716791dc 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -122,9 +122,10 @@ impl MatmulKernel for MatmulCpuKernel {
values,
scales,
group_sums: _,
- scale_group_size: a_group_size,
- code_layout,
+ quantization,
} => {
+ let a_group_size = quantization.scale_group_size();
+ let code_layout = quantization.code_layout();
let compatible = matches!(a_group_size, 32 | 64 | 128)
&& k.is_multiple_of(a_group_size)
&& matches!(b.group_size(), Some(32 | 64 | 128))
@@ -149,7 +150,10 @@ impl MatmulKernel for MatmulCpuKernel {
code_layout,
}
},
- MatmulA::Trellis {
+ MatmulA::PackedHalf {
+ ..
+ }
+ | MatmulA::Trellis {
..
} => {
return Err(MatmulCpuError::UnsupportedLayout {
diff --git a/crates/uzu-engine/src/backends/metal/context.rs b/crates/uzu-engine/src/backends/metal/context.rs
index 57c2b1784..35c8a2873 100644
--- a/crates/uzu-engine/src/backends/metal/context.rs
+++ b/crates/uzu-engine/src/backends/metal/context.rs
@@ -34,8 +34,6 @@ use crate::backends::{
},
};
-pub(super) const LARGE_MIN_GPU_CORES: u32 = 30;
-
pub(super) struct SparseState {
pub did_ops: bool,
pub event_value: u64,
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
index c42e4d23c..4c51fa007 100644
--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
@@ -6,6 +6,7 @@ use derive_more::Display;
pub enum GemmAPrologueKind {
FullPrecision,
Int8Symmetric,
+ PackedHalf,
}
#[repr(C)]
@@ -32,11 +33,17 @@ pub enum GemmTiling {
Tile64x32x256_Simdgroups4x1,
Tile64x64x256_Simdgroups2x2,
Tile128x128x256_Simdgroups4x4,
+ Tile8x32x64_Simdgroups1x2x4,
+ Tile8x32x128_Simdgroups1x2x4,
}
pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
impl GemmTiling {
+ pub const fn is_packed_w4(self) -> bool {
+ matches!(self, Self::Tile8x32x64_Simdgroups1x2x4)
+ }
+
pub const fn block_m(self) -> u32 {
match self {
Self::Tile8x32x32_Simdgroups1x1 => 8,
@@ -51,6 +58,7 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 64,
Self::Tile64x64x256_Simdgroups2x2 => 64,
Self::Tile128x128x256_Simdgroups4x4 => 128,
+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 8,
}
}
@@ -68,6 +76,7 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 32,
Self::Tile64x64x256_Simdgroups2x2 => 64,
Self::Tile128x128x256_Simdgroups4x4 => 128,
+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 32,
}
}
@@ -85,6 +94,8 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 256,
Self::Tile64x64x256_Simdgroups2x2 => 256,
Self::Tile128x128x256_Simdgroups4x4 => 256,
+ Self::Tile8x32x64_Simdgroups1x2x4 => 64,
+ Self::Tile8x32x128_Simdgroups1x2x4 => 128,
}
}
@@ -102,6 +113,7 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 4,
Self::Tile64x64x256_Simdgroups2x2 => 2,
Self::Tile128x128x256_Simdgroups4x4 => 4,
+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 1,
}
}
@@ -119,12 +131,15 @@ impl GemmTiling {
Self::Tile64x32x256_Simdgroups4x1 => 1,
Self::Tile64x64x256_Simdgroups2x2 => 2,
Self::Tile128x128x256_Simdgroups4x4 => 4,
+ Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4 => 2,
}
}
pub const fn simdgroups_k(self) -> u32 {
match self {
- Self::Tile16x32x256_Simdgroups1x1x4 => 4,
+ Self::Tile16x32x256_Simdgroups1x1x4
+ | Self::Tile8x32x64_Simdgroups1x2x4
+ | Self::Tile8x32x128_Simdgroups1x2x4 => 4,
_ => 1,
}
}
@@ -141,6 +156,10 @@ impl GemmTiling {
| Self::Tile128x128x256_Simdgroups4x4
)
}
+
+ pub const fn is_small_batch_variant(self) -> bool {
+ matches!(self, Self::Tile8x32x64_Simdgroups1x2x4 | Self::Tile8x32x128_Simdgroups1x2x4)
+ }
}
pub const fn gemm_tiling_simdgroups_per_row(t: GemmTiling) -> u32 {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
index da31d80d2..2dc3e562d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include "../common/activation_quantization.h"
#include "../common/defines.h"
+#include "../common/packed_half_activation.h"
#include "../common/conversion.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
@@ -16,13 +17,14 @@ using namespace uzu::activation_transform;
#define QUANTIZED (ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)
#define EMITS_GROUP_SUMS (ops == ActivationTransformOp::QuantizeWithGroupSums)
+#define PACKED_HALF (ops == ActivationTransformOp::PackHalf)
template <typename T, typename BiasT>
VARIANTS(T, float, bfloat)
VARIANTS(BiasT, float, bfloat)
PUBLIC KERNEL(ActivationTransform)(
const device T* input OPTIONAL(!in_place),
- device T* fp_out OPTIONAL(!QUANTIZED),
+ device T* fp_out OPTIONAL(!QUANTIZED && !PACKED_HALF),
const device BiasT* bias OPTIONAL(has_bias),
device int8_t* q_out OPTIONAL(QUANTIZED),
device float* scales_out OPTIONAL(QUANTIZED),
@@ -30,6 +32,7 @@ PUBLIC KERNEL(ActivationTransform)(
const device int32_t* rht_factors,
constant uint& batch_size,
constant uint& element_count,
+ device uint* packed_half_out OPTIONAL(PACKED_HALF),
const ActivationTransformOp ops SPECIALIZE,
const bool grouped_by_weight_nibble SPECIALIZE,
const bool in_place SPECIALIZE,
@@ -65,7 +68,7 @@ PUBLIC KERNEL(ActivationTransform)(
values *= float4(factors);
}
- if (!QUANTIZED) {
+ if (!QUANTIZED && !PACKED_HALF) {
if (in_bounds) {
if (has_bias) {
if constexpr (metal::is_same_v<T, bfloat>) {
@@ -78,6 +81,13 @@ PUBLIC KERNEL(ActivationTransform)(
return;
}
+ if (PACKED_HALF) {
+ if (in_bounds) {
+ store_packed_half(values, lane_index, first_index, batch_index, sum_group_size, packed_half_out);
+ }
+ return;
+ }
+
store_quantized_activation_vector(
values,
lane_index,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/packed_half_activation.h b/crates/uzu-engine/src/backends/metal/kernel/common/packed_half_activation.h
new file mode 100644
index 000000000..f60fa6407
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/common/packed_half_activation.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "activation_quantization.h"
+#include "../matmul/common/packed_half_layout.h"
+#include "../matmul/common/quant_unpack.h"
+
+using namespace metal;
+
+UZU_CONST float PACKED_HALF_MAX_FINITE = 65504.0f;
+
+static METAL_FUNC void store_packed_half(
+ const float4 values,
+ const uint lane,
+ const uint first_index,
+ const uint row,
+ const uint group_size,
+ device uint* packed
+) {
+ using Layout32 = uzu::gemm::PackedHalfLayout<32>;
+ using Layout64 = uzu::gemm::PackedHalfLayout<64>;
+ const uint group = first_index / group_size;
+ const uint k = first_index % group_size;
+ const uint group_words = group_size == 32 ? Layout32::GROUP_WORDS : Layout64::GROUP_WORDS;
+ const uint word =
+ group * group_words + (group_size == 32 ? Layout32::word_offset(k, row) : Layout64::word_offset(k, row));
+ device half* halves = reinterpret_cast<device half*>(packed);
+ float2 sums = 0.0f;
+ for (uint i = 0; i < Layout32::VALUES_PER_VECTOR; ++i) {
+ const half value = half(clamp(values[i], -PACKED_HALF_MAX_FINITE, PACKED_HALF_MAX_FINITE));
+ halves[Layout32::ROWS_PER_PAIR * (word + i) + row % Layout32::ROWS_PER_PAIR] = value;
+ sums += float2(float(value), uzu::gemm::w4_pair_bias_value(i) * float(value));
+ }
+ const uint lane_count = group_size / Layout32::VALUES_PER_VECTOR;
+ sums.x = reduce_adjacent_lanes(sums.x, lane_count, [](float x, float y) { return x + y; });
+ sums.y = reduce_adjacent_lanes(sums.y, lane_count, [](float x, float y) { return x + y; });
+ if (lane % lane_count == 0) {
+ const uint sums_offset = group_size == 32 ? Layout32::sums_offset() : Layout64::sums_offset();
+ device float* output = reinterpret_cast<device float*>(packed + group * group_words + sums_offset);
+ output[row * Layout32::SUMS_PER_ROW] = sums.x;
+ output[row * Layout32::SUMS_PER_ROW + 1] = sums.y;
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
index 85b32e5f4..f3ae5f572 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
@@ -2,6 +2,7 @@
#include "../common/activation_quantization.h"
#include "../common/dsl.h"
#include "../common/gated_act_mul.h"
+#include "../common/packed_half_activation.h"
#include "../common/thread_context.h"
#include "../generated/gated_act_mul.h"
#include "../hadamard_transform/hadamard_transform.h"
@@ -16,13 +17,14 @@ using namespace uzu::gated_act_mul;
#define QUANTIZED (ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)
#define EMITS_GROUP_SUMS (ops == GatedActMulOp::QuantizeWithGroupSums)
+#define PACKED_HALF (ops == GatedActMulOp::PackHalf)
template <typename T>
VARIANTS(T, float, bfloat)
PUBLIC KERNEL(GatedActMul) (
const device T* act_operand,
const device T* value_operand OPTIONAL(!interleaved),
- device T* fp_out OPTIONAL(!QUANTIZED),
+ device T* fp_out OPTIONAL(!QUANTIZED && !PACKED_HALF),
device int8_t* q_out OPTIONAL(QUANTIZED),
device float* scales_out OPTIONAL(QUANTIZED),
device int32_t* group_sums_out OPTIONAL(EMITS_GROUP_SUMS),
@@ -37,6 +39,7 @@ PUBLIC KERNEL(GatedActMul) (
const constant float& gate_clip_max OPTIONAL(clip_gate),
const constant float& value_clip_min OPTIONAL(clip_value),
const constant float& value_clip_max OPTIONAL(clip_value),
+ device uint* packed_half_out OPTIONAL(PACKED_HALF),
const GatedActMulOp ops SPECIALIZE,
const bool grouped_by_weight_nibble SPECIALIZE,
const bool interleaved SPECIALIZE,
@@ -109,7 +112,7 @@ PUBLIC KERNEL(GatedActMul) (
results = simdgroup_hadamard_transform_vector(lane_index, results);
}
- if (!QUANTIZED) {
+ if (!QUANTIZED && !PACKED_HALF) {
device T* out_row = fp_out + batch_idx * gated_dim;
if (use_vector_io) {
store_hadamard_vector(out_row + first_index, results);
@@ -121,6 +124,13 @@ PUBLIC KERNEL(GatedActMul) (
return;
}
+ if (PACKED_HALF) {
+ if (valid_count > 0) {
+ store_packed_half(results, lane_index, first_index, batch_idx, sum_group_size, packed_half_out);
+ }
+ return;
+ }
+
store_quantized_activation_vector(
results,
lane_index,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.h b/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.h
index 70797b2f0..5f057e526 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/activation_transform.h
@@ -10,5 +10,6 @@ enum class ActivationTransformOp : uint32_t {
OutputRht = 1,
Quantize = 2,
QuantizeWithGroupSums = 3,
+ PackHalf = 4,
};
} // namespace uzu::activation_transform
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.h
index 571e56256..307a6d667 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gated_act_mul.h
@@ -9,5 +9,6 @@ enum class GatedActMulOp : uint32_t {
FullPrecision = 0,
Quantize = 1,
QuantizeWithGroupSums = 2,
+ PackHalf = 3,
};
} // namespace uzu::gated_act_mul
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
index b555f029c..34d1c9dda 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
@@ -29,6 +29,7 @@ struct GemmDTransform {
enum class GemmAPrologueKind : uint32_t {
FullPrecision = 0,
Int8Symmetric = 1,
+ PackedHalf = 2,
};
enum class GemmBLayout : uint32_t {
@@ -49,6 +50,8 @@ enum class GemmTiling : uint32_t {
Tile64x32x256_Simdgroups4x1 = 9,
Tile64x64x256_Simdgroups2x2 = 10,
Tile128x128x256_Simdgroups4x4 = 11,
+ Tile8x32x64_Simdgroups1x2x4 = 12,
+ Tile8x32x128_Simdgroups1x2x4 = 13,
};
static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
index 2bf74883f..0a939a0f2 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
@@ -303,20 +303,22 @@ struct Fragment {
bool ALIGNED_N = false,
class Ptr,
class FragmentRowStride = Int<1>,
- class FragmentColStride = Int<1>>
+ class FragmentColStride = Int<1>,
+ class ColStride = Int<1>>
METAL_FUNC void store_safe(
const ushort simd_lane_id,
Ptr destination,
- const int leading_dimension,
+ const int row_stride,
const short2 tile_dimensions,
FragmentRowStride fragment_row_stride = {},
- FragmentColStride fragment_col_stride = {}
+ FragmentColStride fragment_col_stride = {},
+ ColStride col_stride = {}
) thread {
transfer<STORE, SAFE, ALIGNED_M, ALIGNED_N>(
simd_lane_id,
destination,
- leading_dimension,
- Int<1>{},
+ row_stride,
+ col_stride,
tile_dimensions.y,
tile_dimensions.x,
fragment_row_stride,
@@ -360,6 +362,38 @@ struct Fragment {
}
}
+ template <ushort K_SLICES, ushort OUTPUT_GROUPS>
+ METAL_FUNC bool reduce_k_slices(
+ threadgroup ElementType* partials,
+ const thread ThreadContext& thread_context
+ ) thread {
+ if constexpr (K_SLICES == 1) {
+ return true;
+ } else {
+ const ushort k_slice = ushort(thread_context.simdgroup_index) / OUTPUT_GROUPS;
+ const ushort output_group = ushort(thread_context.simdgroup_index) % OUTPUT_GROUPS;
+ const ushort lane = ushort(thread_context.simd_lane_id);
+ thread ElementType* values = elements();
+ constexpr ushort ELEMENTS = ELEMENTS_PER_FRAGMENT;
+ constexpr ushort GROUP_STRIDE = (K_SLICES - 1) * ELEMENTS * METAL_SIMD_SIZE;
+ if (k_slice != 0) {
+ for (ushort e = 0; e < ELEMENTS; ++e) {
+ partials[output_group * GROUP_STRIDE + ((k_slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane] = values[e];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (k_slice != 0) {
+ return false;
+ }
+ for (ushort slice = 1; slice < K_SLICES; ++slice) {
+ for (ushort e = 0; e < ELEMENTS; ++e) {
+ values[e] += partials[output_group * GROUP_STRIDE + ((slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane];
+ }
+ }
+ return true;
+ }
+ }
+
private:
UZU_CONST bool LOAD = true;
UZU_CONST bool STORE = false;
@@ -435,6 +469,21 @@ private:
}
};
+template <typename T, ushort ROWS, ushort COLS, class Ops, class Generator>
+struct GeneratedFragment {
+ using ElementType = T;
+ using FragmentOpsType = Ops;
+ using ThreadVectorType = typename Ops::template ThreadVector<T>;
+ UZU_CONST ushort ROW_FRAGMENTS = ROWS;
+ UZU_CONST ushort COL_FRAGMENTS = COLS;
+ UZU_CONST bool MMA_TRANSPOSE = false;
+ Generator generator;
+
+ METAL_FUNC ThreadVectorType fragment_at(const ushort row, const ushort col) const thread {
+ return generator(row, col);
+ }
+};
+
template <class Ops, class Read>
struct OperandFragmentTraits {
UZU_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/packed_half_layout.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/packed_half_layout.h
new file mode 100644
index 000000000..40fc02d49
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/packed_half_layout.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <metal_stdlib>
+#include "../../common/defines.h"
+
+using namespace metal;
+
+namespace uzu::gemm {
+
+template <uint GROUP_SIZE>
+struct PackedHalfLayout {
+ UZU_CONST uint MMA_WIDTH = 8;
+ UZU_CONST uint VALUES_PER_VECTOR = 4;
+ UZU_CONST uint ROWS_PER_PAIR = 2;
+ UZU_CONST uint SUMS_PER_ROW = 2;
+ UZU_CONST uint G64_K_INTERLEAVE = 2;
+ UZU_CONST uint G64_K_CHUNK = G64_K_INTERLEAVE * MMA_WIDTH;
+ UZU_CONST uint STEPS = GROUP_SIZE / MMA_WIDTH;
+ UZU_CONST uint GROUP_WORDS = METAL_SIMD_SIZE * STEPS + (METAL_SIMD_SIZE / MMA_WIDTH) * ROWS_PER_PAIR * SUMS_PER_ROW;
+
+ static METAL_FUNC uint pair_offset(uint slot, uint fm) { return (slot * MMA_WIDTH + fm) * STEPS; }
+ static METAL_FUNC uint codes_offset(uint group, uint slot) {
+ return group * (GROUP_SIZE / 2) + slot * (GROUP_SIZE / 8);
+ }
+ static METAL_FUNC uint sums_offset() { return METAL_SIMD_SIZE * STEPS; }
+ static METAL_FUNC uint word_offset(uint k, uint row) {
+ const uint chunk = (k % G64_K_CHUNK) / VALUES_PER_VECTOR;
+ const uint fm =
+ GROUP_SIZE == 32 ? k / VALUES_PER_VECTOR : (k / G64_K_CHUNK) * G64_K_INTERLEAVE + chunk % G64_K_INTERLEAVE;
+ const uint step = GROUP_SIZE == 32 ? k % VALUES_PER_VECTOR
+ : (chunk / G64_K_INTERLEAVE) * VALUES_PER_VECTOR + k % VALUES_PER_VECTOR;
+ return pair_offset(row / ROWS_PER_PAIR, fm) + step;
+ }
+};
+
+} // namespace uzu::gemm
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index 4524c4d93..ffe4ce189 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -31,6 +31,58 @@ UZU_CONST ushort W4_BITS = 4;
UZU_CONST uint W4_SIGN_MASK = symmetric_zero_point<W4_BITS>() * 0x11111111u;
UZU_CONST uint W8_SIGN_MASK = symmetric_zero_point<8>() * 0x01010101u;
UZU_CONST uint W4_NIBBLE_MASK = 0x0F0F0F0Fu;
+UZU_CONST uint ZERO_POINTS_PER_BYTE = 2;
+UZU_CONST uint W8_MAGIC_FLOAT_OFFSET = 0x4B400000u;
+
+UZU_CONST uint W4_PAIR_HALF_BITS = 16;
+UZU_CONST uint W4_PAIR_MANTISSA_BITS = 10;
+UZU_CONST uint W4_PAIR_EXPONENT_BIAS = 15;
+
+static METAL_FUNC constexpr uint w4_pair_offset(uint component) {
+ constexpr uint COMPONENTS = W4_PAIR_HALF_BITS / W4_BITS;
+ return W4_BITS * (component < COMPONENTS ? component : COMPONENTS - 1);
+}
+
+static METAL_FUNC constexpr uint w4_pair_position(uint component) {
+ const uint offset = w4_pair_offset(component);
+ return offset + W4_BITS <= W4_PAIR_MANTISSA_BITS ? offset : offset - (W4_PAIR_HALF_BITS - W4_PAIR_MANTISSA_BITS);
+}
+
+template <uint COMPONENT>
+static METAL_FUNC half2 w4_pair_at(uint word) {
+ constexpr uint offset = w4_pair_offset(COMPONENT);
+ constexpr uint position = w4_pair_position(COMPONENT);
+ constexpr uint mask = ((1u << W4_BITS) - 1) << position;
+ constexpr uint exponent = (W4_PAIR_EXPONENT_BIAS + W4_PAIR_MANTISSA_BITS - position) << W4_PAIR_MANTISSA_BITS;
+ return as_type<half2>(
+ ((word >> (offset - position)) & (mask | (mask << W4_PAIR_HALF_BITS))) |
+ (exponent | (exponent << W4_PAIR_HALF_BITS))
+ );
+}
+
+static METAL_FUNC constexpr float w4_pair_bias_value(uint component) {
+ return float(1u << (W4_PAIR_MANTISSA_BITS - w4_pair_position(component)));
+}
+
+static METAL_FUNC half2 w4_pair(uint word, ushort component) {
+ switch (component) {
+ case 0:
+ return w4_pair_at<0>(word);
+ case 1:
+ return w4_pair_at<1>(word);
+ case 2:
+ return w4_pair_at<2>(word);
+ default:
+ return w4_pair_at<3>(word);
+ }
+}
+
+template <uint ORIGIN>
+METAL_FUNC float magic_code_float(uint code) {
+ return as_type<float>(code + W8_MAGIC_FLOAT_OFFSET - ORIGIN);
+}
+
+METAL_FUNC float magic_dequant_bias(float scale) { return -12582912.0f * scale; }
template <typename U, int N, int bits>
inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local, const bool signed_codes) {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
index 77161f5ca..89528b0de 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
@@ -18,6 +18,8 @@ constexpr uint gemm_tiling_block_m(GemmTiling t) {
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 64
: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 8
+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 8
: 0;
}
@@ -34,6 +36,8 @@ constexpr uint gemm_tiling_block_n(GemmTiling t) {
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 32
: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 32
+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 32
: 0;
}
@@ -50,6 +54,8 @@ constexpr uint gemm_tiling_block_k(GemmTiling t) {
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 256
: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 256
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 256
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 64
+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 128
: 0;
}
@@ -66,6 +72,8 @@ constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 4
: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 1
+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 1
: 0;
}
@@ -82,11 +90,16 @@ constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {
: t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 1
: t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2
: t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 2
+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 2
: 0;
}
constexpr uint gemm_tiling_simdgroups_per_k(GemmTiling t) {
- return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4 : 1;
+ return t == GemmTiling::Tile16x32x256_Simdgroups1x1x4 ? 4
+ : t == GemmTiling::Tile8x32x64_Simdgroups1x2x4 ? 4
+ : t == GemmTiling::Tile8x32x128_Simdgroups1x2x4 ? 4
+ : 1;
}
} // namespace gemm
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
index 681597107..2c4106a84 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/operands.h
@@ -3,6 +3,8 @@
#include <metal_stdlib>
#include "../../../generated/gemm.h"
+#include "../../../generated/matmul.h"
+#include "../../common/quant_unpack.h"
using namespace metal;
@@ -13,6 +15,7 @@ namespace operands {
template <GemmAPrologueKind PROLOGUE, typename Element, ushort ACTIVATION_GROUP_SIZE, bool CODES_GROUPED_BY_NIBBLE>
struct LeftOperand {
UZU_CONST bool QUANTIZED = PROLOGUE == GemmAPrologueKind::Int8Symmetric;
+ UZU_CONST bool PACKED_HALF = PROLOGUE == GemmAPrologueKind::PackedHalf;
UZU_CONST ushort BITS = QUANTIZED ? 8 : 0;
UZU_CONST ushort GROUP_SIZE = QUANTIZED ? ACTIVATION_GROUP_SIZE : 0;
UZU_CONST bool GROUPED_BY_NIBBLE = CODES_GROUPED_BY_NIBBLE;
@@ -73,6 +76,7 @@ struct LeftStorage {
const device int8_t* codes;
const device float* scales;
const device int32_t* group_sums;
+ const device uint* packed_half;
METAL_FUNC const device int32_t* correction_sums() const thread {
static_assert(Left::QUANTIZED, "correction_sums is only valid for int8 activations");
@@ -90,6 +94,19 @@ struct RightStorage {
bool signed_codes;
GemmBLayout gemm_b_layout;
+ METAL_FUNC uint sign_mask() const thread {
+ return signed_codes ? (Right::BITS == 4 ? W4_SIGN_MASK : W8_SIGN_MASK) : 0u;
+ }
+
+ template <typename Index>
+ METAL_FUNC const device uint8_t* row_codes(
+ const Index row,
+ const constant uzu::matmul::GemmParams* params
+ ) const thread {
+ const Index row_stride = Index(params->K) / (8 / Right::BITS);
+ return codes + row * row_stride;
+ }
+
METAL_FUNC const device typename Right::ScaleElement* bias() const thread {
static_assert(Right::SCHEME == GemmBPrologueKind::ScaleBiasDequant, "bias is only valid for ScaleBiasDequant");
return biases;
@@ -102,6 +119,31 @@ struct RightStorage {
);
return zero_points;
}
+
+ template <typename Index>
+ METAL_FUNC Index
+ scale_index(const Index row, const Index group, const constant uzu::matmul::GemmParams* params) const thread {
+ return row * Index(params->scale_output_stride) + group * Index(params->scale_group_stride);
+ }
+
+ METAL_FUNC typename Right::ScaleElement scale(
+ const uint row,
+ const uint group,
+ const constant uzu::matmul::GemmParams* params
+ ) const thread {
+ return scales[scale_index(row, group, params)];
+ }
+
+ METAL_FUNC uint
+ zero_point_index(const uint row, const uint group, const constant uzu::matmul::GemmParams* params) const thread {
+ return row * params->zero_point_output_stride + group * params->zero_point_group_stride;
+ }
+
+ METAL_FUNC uint
+ zero_point(const uint row, const uint group, const constant uzu::matmul::GemmParams* params) const thread {
+ const uint index = zero_point_index(row, group, params);
+ return decode_zero_point<Right::BITS>(zp() + index / ZERO_POINTS_PER_BYTE, index % ZERO_POINTS_PER_BYTE);
+ }
};
template <typename Left, typename Element>
@@ -109,12 +151,15 @@ METAL_FUNC LeftStorage<Left> pack_left(
const device Element* a,
const device int8_t* codes,
const device float* scales,
- const device int32_t* group_sums
+ const device int32_t* group_sums,
+ const device uint* packed_half
) {
- if constexpr (Left::QUANTIZED) {
- return {nullptr, codes, scales, group_sums};
+ if constexpr (Left::PACKED_HALF) {
+ return {nullptr, nullptr, nullptr, nullptr, packed_half};
+ } else if constexpr (Left::QUANTIZED) {
+ return {nullptr, codes, scales, group_sums, nullptr};
} else {
- return {a, nullptr, nullptr, nullptr};
+ return {a, nullptr, nullptr, nullptr, nullptr};
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/direct.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/direct.h
new file mode 100644
index 000000000..555ee27a5
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/direct.h
@@ -0,0 +1,166 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../../../common/conversion.h"
+#include "../../../../../generated/gemm.h"
+#include "../../../../common/fragment.h"
+#include "../../../../common/quant_unpack.h"
+#include "../../../../common/simdgroup_fragment_ops.h"
+#include "../../operands.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+namespace schedules {
+
+struct DirectSchedule {
+ UZU_CONST bool TRANSPOSED_OUTPUT = false;
+
+ template <typename Core>
+ static METAL_FUNC typename Core::AccumFragment launch(
+ const typename Core::LeftStorage left,
+ const typename Core::RightStorage right,
+ const constant uzu::matmul::GemmParams* params,
+ const thread typename Core::TileContext& tile_context
+ ) {
+ using RightOperand = typename Core::Right;
+ constexpr ushort K_STEPS_PER_BLOCK = 16;
+ using Ops = typename Core::FragmentOps;
+ constexpr uint K_BLOCK = Core::THREADGROUP_BLOCK_K;
+ constexpr uint ACTIVATION_VECTORS_PER_LANE = K_STEPS_PER_BLOCK / 4;
+
+ const uint lane_row = uint(tile_context.lane_position.y);
+ const uint lane_column = uint(tile_context.lane_position.x);
+ const uint inner_dimension = params->K;
+ const uint last_weight_row = params->N - 1;
+ const uint weight_sign_mask = right.sign_mask();
+
+ const uint simdgroup_column_base = tile_context.simdgroup_column_base;
+ uint weight_rows[Core::TILES_N][2];
+ METAL_PRAGMA_UNROLL
+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {
+ METAL_PRAGMA_UNROLL
+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {
+ weight_rows[tile_n][row_pair] =
+ min(simdgroup_column_base + Ops::FRAGMENT_COLS * tile_n + lane_column + row_pair, last_weight_row);
+ }
+ }
+
+ typename Core::AccumFragment accumulator;
+ accumulator.clear();
+
+ const device bfloat* activation_row =
+ left.values + min(tile_context.block_geometry.block_row_start + lane_row, params->M - 1) * inner_dimension +
+ K_STEPS_PER_BLOCK * lane_column;
+ const uint k_block_end = tile_context.k_block_end;
+
+ uint4 activation_words[ACTIVATION_VECTORS_PER_LANE];
+ uint4 weight_words[Core::TILES_N][2];
+ float weight_scales[Core::TILES_N][2];
+ float dequant_offsets[Core::TILES_N][2];
+
+ auto load_activations = [&](uint k_block, thread uint4(&activation_dst)[ACTIVATION_VECTORS_PER_LANE]) {
+ const device uint4* source = reinterpret_cast<const device uint4*>(activation_row + k_block * K_BLOCK);
+ METAL_PRAGMA_UNROLL
+ for (uint vector = 0; vector < ACTIVATION_VECTORS_PER_LANE; ++vector) {
+ activation_dst[vector] = source[vector];
+ }
+ };
+ auto load_weights = [&](uint k_block, thread uint4(&weight_dst)[Core::TILES_N][2]) {
+ const uint weight_k_lane = k_block * K_BLOCK + K_STEPS_PER_BLOCK * lane_row;
+ METAL_PRAGMA_UNROLL
+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {
+ METAL_PRAGMA_UNROLL
+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {
+ weight_dst[tile_n][row_pair] = *reinterpret_cast<const device uint4*>(
+ right.row_codes(weight_rows[tile_n][row_pair], params) + weight_k_lane
+ );
+ }
+ }
+ };
+ auto load_metadata =
+ [&](uint k_block, thread float (&scale_dst)[Core::TILES_N][2], thread float (&offset_dst)[Core::TILES_N][2]) {
+ const uint weight_group_index = (k_block * K_BLOCK + K_STEPS_PER_BLOCK * lane_row) / RightOperand::GROUP_SIZE;
+ METAL_PRAGMA_UNROLL
+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {
+ METAL_PRAGMA_UNROLL
+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {
+ const uint row = weight_rows[tile_n][row_pair];
+ scale_dst[tile_n][row_pair] = float(right.scale(row, weight_group_index, params));
+ offset_dst[tile_n][row_pair] = magic_dequant_bias(scale_dst[tile_n][row_pair]);
+ }
+ }
+ };
+
+ uint k_block = tile_context.k_block_begin + tile_context.simdgroup_k_slice;
+ if (k_block < k_block_end) {
+ load_activations(k_block, activation_words);
+ load_weights(k_block, weight_words);
+ load_metadata(k_block, weight_scales, dequant_offsets);
+ }
+ for (; k_block < k_block_end; k_block += Core::SIMDGROUPS_PER_K) {
+ const uint next_k_block = k_block + Core::SIMDGROUPS_PER_K;
+ const uint prefetch_k_block = next_k_block < k_block_end ? next_k_block : k_block;
+ METAL_PRAGMA_UNROLL
+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {
+ METAL_PRAGMA_UNROLL
+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {
+ weight_words[tile_n][row_pair] ^= uint4(weight_sign_mask);
+ }
+ }
+
+ auto activation_generator = [&](ushort, ushort k_index) -> float2 {
+ const uint word0 = reinterpret_cast<const thread uint*>(activation_words)[k_index / 2];
+ const uint word1 = reinterpret_cast<const thread uint*>(activation_words)[K_STEPS_PER_BLOCK / 2 + k_index / 2];
+ const float2 values0 = uzu::unpack_bfloat(word0);
+ const float2 values1 = uzu::unpack_bfloat(word1);
+ return k_index % 2 ? float2(values0.y, values1.y) : float2(values0.x, values1.x);
+ };
+ auto weight_generator = [&](ushort k_index, ushort tile_n) -> float2 {
+ const uint word_index = k_index * RightOperand::BITS / 32;
+ auto decode = [&](ushort row_pair) {
+ return fma(
+ magic_code_float<symmetric_zero_point<8>()>(
+ extract_bits(weight_words[tile_n][row_pair][word_index], 8 * (k_index % 4), 8)
+ ),
+ weight_scales[tile_n][row_pair],
+ dequant_offsets[tile_n][row_pair]
+ );
+ };
+ return float2(decode(0), decode(1));
+ };
+ using ActivationFragment =
+ uzu::matmul::GeneratedFragment<float, Core::TILES_M, K_STEPS_PER_BLOCK, Ops, decltype(activation_generator)>;
+ using WeightFragment =
+ uzu::matmul::GeneratedFragment<float, K_STEPS_PER_BLOCK, Core::TILES_N, Ops, decltype(weight_generator)>;
+ ActivationFragment activation_fragment{activation_generator};
+ WeightFragment weight_fragment{weight_generator};
+ uzu::matmul::fragment_mma(accumulator, activation_fragment, weight_fragment);
+
+ uint4 next_weight_words[Core::TILES_N][2];
+ float next_weight_scales[Core::TILES_N][2];
+ float next_dequant_offsets[Core::TILES_N][2];
+ // Keep next operands separate so their loads can overlap the current MMA.
+ load_activations(prefetch_k_block, activation_words);
+ load_metadata(prefetch_k_block, next_weight_scales, next_dequant_offsets);
+ load_weights(prefetch_k_block, next_weight_words);
+ METAL_PRAGMA_UNROLL
+ for (uint tile_n = 0; tile_n < Core::TILES_N; ++tile_n) {
+ METAL_PRAGMA_UNROLL
+ for (uint row_pair = 0; row_pair < 2; ++row_pair) {
+ weight_scales[tile_n][row_pair] = next_weight_scales[tile_n][row_pair];
+ dequant_offsets[tile_n][row_pair] = next_dequant_offsets[tile_n][row_pair];
+ weight_words[tile_n][row_pair] = next_weight_words[tile_n][row_pair];
+ }
+ }
+ }
+
+ return accumulator;
+ }
+};
+
+} // namespace schedules
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h
new file mode 100644
index 000000000..3f6df944c
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/small_batch/packed.h
@@ -0,0 +1,131 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../../../generated/gemm.h"
+#include "../../../../common/fragment.h"
+#include "../../../../common/quant_unpack.h"
+#include "../../../../common/simdgroup_fragment_ops.h"
+#include "../../../../common/packed_half_layout.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+namespace schedules {
+
+struct PackedSchedule {
+ UZU_CONST bool TRANSPOSED_OUTPUT = true;
+
+ template <ushort WORDS_PER_LANE>
+ struct WeightDecoder {
+ thread const uint* packed_words;
+
+ METAL_FUNC half2 operator()(const ushort row, const ushort k) const thread {
+ return w4_pair(packed_words[row * WORDS_PER_LANE + k / 4], k % 4);
+ }
+ };
+
+ template <typename Core>
+ static METAL_FUNC typename Core::AccumFragment launch(
+ const typename Core::LeftStorage left,
+ const typename Core::RightStorage right,
+ const constant uzu::matmul::GemmParams* params,
+ const thread typename Core::TileContext& tile_context
+ ) {
+ using RightOperand = typename Core::Right;
+ constexpr bool HAS_BIAS = RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant;
+ constexpr ushort WEIGHT_GROUP_SIZE = RightOperand::GROUP_SIZE;
+ constexpr ushort K_STEPS_PER_GROUP = PackedHalfLayout<WEIGHT_GROUP_SIZE>::STEPS;
+ constexpr ushort WORDS_PER_LANE = WEIGHT_GROUP_SIZE / 32;
+ constexpr ushort WORDS_PER_GROUP = PackedHalfLayout<WEIGHT_GROUP_SIZE>::GROUP_WORDS;
+ constexpr ushort GROUPS_PER_K_BLOCK = Core::THREADGROUP_BLOCK_K / WEIGHT_GROUP_SIZE;
+ using Ops = typename Core::FragmentOps;
+ using WeightFragment =
+ uzu::matmul::GeneratedFragment<half, Core::TILES_M, K_STEPS_PER_GROUP, Ops, WeightDecoder<WORDS_PER_LANE>>;
+ using ActivationFragment = uzu::matmul::Fragment<half, K_STEPS_PER_GROUP, Core::TILES_N, Ops>;
+
+ const device uint* packed_activations = left.packed_half;
+ const uint fragment_column_pair = uint(tile_context.lane_position.x) / 2;
+ const uint lane_row = uint(tile_context.lane_position.y);
+ const uint weight_sign_mask = right.sign_mask();
+
+ typename Core::AccumFragment accumulator;
+ accumulator.clear();
+ const uint k_block_end = tile_context.k_block_end;
+ for (uint k_block = tile_context.k_block_begin + tile_context.simdgroup_k_slice; k_block < k_block_end;
+ k_block += Core::SIMDGROUPS_PER_K) {
+ const uint first_weight_group = k_block * GROUPS_PER_K_BLOCK;
+ for (ushort group_in_block = 0; group_in_block < GROUPS_PER_K_BLOCK; ++group_in_block) {
+ const uint weight_group_index = first_weight_group + group_in_block;
+ const device uint* packed_group = packed_activations + weight_group_index * WORDS_PER_GROUP;
+ const device uint* activation_pairs =
+ packed_group + PackedHalfLayout<WEIGHT_GROUP_SIZE>::pair_offset(fragment_column_pair, lane_row);
+ ActivationFragment activation_fragment;
+ METAL_PRAGMA_UNROLL
+ for (ushort vector = 0; vector < K_STEPS_PER_GROUP / 4; ++vector) {
+ const uint4 raw = reinterpret_cast<const device uint4*>(activation_pairs)[vector];
+ activation_fragment.fragment_at(4 * vector, 0) = as_type<half2>(raw.x);
+ activation_fragment.fragment_at(4 * vector + 1, 0) = as_type<half2>(raw.y);
+ activation_fragment.fragment_at(4 * vector + 2, 0) = as_type<half2>(raw.z);
+ activation_fragment.fragment_at(4 * vector + 3, 0) = as_type<half2>(raw.w);
+ }
+ const float4 activation_sums = reinterpret_cast<const device float4*>(
+ packed_group + PackedHalfLayout<WEIGHT_GROUP_SIZE>::sums_offset()
+ )[fragment_column_pair];
+
+ uint weight_rows[Core::TILES_M];
+ float weight_scales[Core::TILES_M];
+ float weight_offsets[Core::TILES_M]; // zero point, or bias
+ uint packed_weight_words[Core::TILES_M * WORDS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (ushort tile_m = 0; tile_m < Core::TILES_M; ++tile_m) {
+ weight_rows[tile_m] =
+ min(tile_context.simdgroup_column_base + Ops::FRAGMENT_ROWS * tile_m + lane_row, params->N - 1);
+ const device uint* weight_source = reinterpret_cast<const device uint*>(
+ right.row_codes(weight_rows[tile_m], params) +
+ PackedHalfLayout<WEIGHT_GROUP_SIZE>::codes_offset(weight_group_index, fragment_column_pair)
+ );
+ METAL_PRAGMA_UNROLL
+ for (ushort word_index = 0; word_index < WORDS_PER_LANE; ++word_index) {
+ packed_weight_words[tile_m * WORDS_PER_LANE + word_index] = weight_source[word_index] ^ weight_sign_mask;
+ }
+ weight_scales[tile_m] = float(right.scale(weight_rows[tile_m], weight_group_index, params));
+ if constexpr (HAS_BIAS) {
+ weight_offsets[tile_m] =
+ float(right.bias()[right.scale_index(weight_rows[tile_m], weight_group_index, params)]);
+ } else {
+ weight_offsets[tile_m] = float(right.zero_point(weight_rows[tile_m], weight_group_index, params));
+ }
+ }
+
+ WeightDecoder<WORDS_PER_LANE> weight_decoder{packed_weight_words};
+ WeightFragment weight_fragment{weight_decoder};
+ typename Core::AccumFragment group_product;
+ group_product.clear();
+ uzu::matmul::fragment_mma(group_product, weight_fragment, activation_fragment);
+ accumulator.for_each_lane(
+ [&](ushort tile_m, ushort input_index, thread float& accumulator_value, thread float& group_product_value) {
+ if constexpr (HAS_BIAS) {
+ accumulator_value =
+ fma(group_product_value - activation_sums[2 * input_index + 1],
+ weight_scales[tile_m],
+ fma(weight_offsets[tile_m], activation_sums[2 * input_index], accumulator_value));
+ } else {
+ const float correction =
+ fma(weight_offsets[tile_m], activation_sums[2 * input_index], activation_sums[2 * input_index + 1]);
+ accumulator_value = fma(group_product_value - correction, weight_scales[tile_m], accumulator_value);
+ }
+ },
+ group_product
+ );
+ }
+ }
+
+ return accumulator;
+ }
+};
+
+} // namespace schedules
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h
new file mode 100644
index 000000000..eeb6f272b
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/small_batch_mma_core.h
@@ -0,0 +1,182 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../common/thread_context.h"
+#include "../../../generated/matmul.h"
+#include "../generated/gemm.h"
+#include "../../../common/soft_cap.h"
+#include "../../common/fragment.h"
+#include "../../common/simdgroup_fragment_ops.h"
+#include "block_geometry.h"
+#include "gemm_rht.h"
+#include "gemm_tiling.h"
+#include "operands.h"
+#include "schedules/small_batch/packed.h"
+#include "schedules/small_batch/direct.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <typename OutputElementType_, GemmTiling GEMM_TILING, typename LeftOperand, typename RightOperand>
+struct SmallBatchMmaCore {
+ using OutputElementType = OutputElementType_;
+ using Left = LeftOperand;
+ using Right = RightOperand;
+ using RightElementType = typename Right::ElementType;
+ using LeftStorage = operands::LeftStorage<Left>;
+ using RightStorage = operands::RightStorage<Right>;
+ using Schedule = metal::conditional_t<Left::PACKED_HALF, schedules::PackedSchedule, schedules::DirectSchedule>;
+ using FragmentOps = uzu::matmul::SimdgroupFragmentOps;
+ UZU_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
+ UZU_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
+ UZU_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);
+ UZU_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
+ UZU_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
+ UZU_CONST int SIMDGROUPS_PER_K = gemm_tiling_simdgroups_per_k(GEMM_TILING);
+ UZU_CONST int SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
+ UZU_CONST int SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
+ UZU_CONST ushort TILES_M = Schedule::TRANSPOSED_OUTPUT ? SIMDGROUP_BLOCK_N / FragmentOps::FRAGMENT_ROWS
+ : SIMDGROUP_BLOCK_M / FragmentOps::FRAGMENT_ROWS;
+ UZU_CONST ushort TILES_N = Schedule::TRANSPOSED_OUTPUT ? SIMDGROUP_BLOCK_M / FragmentOps::FRAGMENT_COLS
+ : SIMDGROUP_BLOCK_N / FragmentOps::FRAGMENT_COLS;
+ using AccumFragment = uzu::matmul::Fragment<float, TILES_M, TILES_N, FragmentOps>;
+
+ using BlockGeometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>;
+ struct TileContext {
+ BlockGeometry block_geometry;
+ ushort simd_lane_id;
+ short2 lane_position;
+ uint k_partition;
+ ushort simdgroup_k_slice;
+ uint simdgroup_column_base;
+ uint k_block_begin;
+ uint k_block_end;
+ };
+
+ static METAL_FUNC TileContext
+ make_tile_context(const constant uzu::matmul::GemmParams* params, const thread ThreadContext& thread_context) {
+ const ushort simd_lane_id = ushort(thread_context.simd_lane_id);
+ const uint2 id =
+ tile_id(uint2(thread_context.threadgroup_position.x, thread_context.threadgroup_position.y), params);
+ const BlockGeometry block_geometry = BlockGeometry::compute(id, params);
+ const ushort simdgroup_index = ushort(thread_context.simdgroup_index);
+ const uint total_blocks = params->K / THREADGROUP_BLOCK_K;
+ const uint blocks_per_partition =
+ SIMDGROUPS_PER_K * params->aligned_inner_iterations + total_blocks % SIMDGROUPS_PER_K;
+ const uint k_block_begin = thread_context.threadgroup_position.z * blocks_per_partition;
+ return {
+ block_geometry,
+ simd_lane_id,
+ FragmentOps::get_position(simd_lane_id),
+ thread_context.threadgroup_position.z,
+ ushort(simdgroup_index / SIMDGROUPS_PER_COLUMN),
+ block_geometry.block_col_start + uint(simdgroup_index % SIMDGROUPS_PER_COLUMN) * SIMDGROUP_BLOCK_N,
+ k_block_begin,
+ k_block_begin + blocks_per_partition
+ };
+ }
+
+ static METAL_FUNC void run(
+ LeftStorage left,
+ RightStorage right,
+ device OutputElementType* d,
+ const constant uzu::matmul::GemmParams* params,
+ GemmDTransform output_transform,
+ const constant float& soft_cap,
+ const device RightElementType* output_bias,
+ const device int32_t* rht_factors,
+ threadgroup RightElementType* scratch,
+ const thread ThreadContext& thread_context
+ ) {
+ TileContext tile_context = make_tile_context(params, thread_context);
+ if (tile_context.block_geometry.out_of_bounds) {
+ return;
+ }
+ AccumFragment accumulator = Schedule::template launch<SmallBatchMmaCore>(left, right, params, tile_context);
+ const bool output_writer = accumulator.template reduce_k_slices<SIMDGROUPS_PER_K, SIMDGROUPS_PER_COLUMN>(
+ reinterpret_cast<threadgroup float*>(scratch),
+ thread_context
+ );
+ const bool needs_rht = output_transform.contains(GemmDTransform::RHT);
+ if (!output_writer && !needs_rht) {
+ return;
+ }
+
+ const uint row_start = tile_context.block_geometry.block_row_start;
+ const uint m_remaining = min(uint(THREADGROUP_BLOCK_M), params->M - row_start);
+ const uint n_remaining = tile_context.simdgroup_column_base < params->N
+ ? min(uint(SIMDGROUP_BLOCK_N), params->N - tile_context.simdgroup_column_base)
+ : 0;
+ const uint safe_output_col = min(tile_context.simdgroup_column_base, params->N - 1);
+ const short2 tile_dimensions = Schedule::TRANSPOSED_OUTPUT ? short2(short(m_remaining), short(n_remaining))
+ : short2(short(n_remaining), short(m_remaining));
+ device OutputElementType* row_destination =
+ d + size_t(tile_context.k_partition) * size_t(params->M) * size_t(params->N) +
+ size_t(row_start) * params->leading_dimension_d;
+ if (output_writer) {
+ if (output_transform.contains(GemmDTransform::SCALE)) {
+ const uzu::matmul::TransformScale<float, float> scale(params->ab_scale);
+ accumulator.map([&](float value) { return scale.apply(value); });
+ }
+ const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);
+ if (needs_bias && !needs_rht) {
+ accumulator.map_coords(tile_context.simd_lane_id, [&](short row, short col, float value) {
+ const uint bias_col = Schedule::TRANSPOSED_OUTPUT ? uint(row) : uint(col);
+ return bias_col < n_remaining ? value + float(output_bias[tile_context.simdgroup_column_base + bias_col])
+ : value;
+ });
+ }
+ if (output_transform.contains(GemmDTransform::SOFT_CAP)) {
+ accumulator.map([&](float value) { return uzu::apply_soft_cap(value, soft_cap); });
+ }
+
+ device OutputElementType* destination = row_destination + safe_output_col;
+ const int row_stride = Schedule::TRANSPOSED_OUTPUT ? 1 : int(params->leading_dimension_d);
+ const auto col_stride = [&]() {
+ if constexpr (Schedule::TRANSPOSED_OUTPUT) {
+ return int(params->leading_dimension_d);
+ } else {
+ return uzu::Int<1>{};
+ }
+ }();
+ accumulator.store_safe(
+ tile_context.simd_lane_id,
+ destination,
+ row_stride,
+ tile_dimensions,
+ uzu::Int<1>{},
+ uzu::Int<1>{},
+ col_stride
+ );
+ }
+
+ if (needs_rht) {
+ const uint tile_col_start = tile_context.block_geometry.block_col_start;
+ const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(tile_col_start)));
+ device OutputElementType* tile_output = row_destination + tile_col_start;
+ const device RightElementType* tile_bias = output_bias;
+ if (output_transform.contains(GemmDTransform::BIAS)) {
+ tile_bias += tile_col_start;
+ }
+
+ threadgroup_barrier(mem_flags::mem_device);
+ apply_output_random_hadamard_transform(
+ tile_output,
+ rht_factors + tile_col_start,
+ tile_bias,
+ output_transform.contains(GemmDTransform::BIAS),
+ ushort(m_remaining),
+ tile_block_cols,
+ params->leading_dimension_d,
+ SIMDGROUPS_PER_K * SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN,
+ thread_context
+ );
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 19f87e0bd..5c7f457ff 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -6,15 +6,21 @@
#include "common/gemm_tiling.h"
#include "common/mxu_mma_core.h"
#include "common/simdgroup_mma_core.h"
+#include "common/small_batch_mma_core.h"
using namespace metal;
using namespace uzu::gemm;
#define A_IS_INT8 (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric)
+#define GEMM_PACKED_HALF_TILING (GEMM_TILING == GemmTiling::Tile8x32x64_Simdgroups1x2x4)
+#define GEMM_SMALL_BATCH (GEMM_PACKED_HALF_TILING || GEMM_TILING == GemmTiling::Tile8x32x128_Simdgroups1x2x4)
+#define GEMM_SMALL_BATCH_W8 (GEMM_SMALL_BATCH && !GEMM_PACKED_HALF_TILING)
#define NEEDS_ASYMMETRIC_WEIGHT_CORRECTION (A_IS_INT8 && B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant)
#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision && !A_IS_INT8)
#define GEMM_TGA_ELEMENTS \
- ((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
+ ((GEMM_SMALL_BATCH || USE_MXU) \
+ ? 1 \
+ : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
#define GEMM_K_PARTIAL_ELEMENTS \
(gemm_tiling_simdgroups_per_k(GEMM_TILING) == 1 \
? 1 \
@@ -25,7 +31,9 @@ using namespace uzu::gemm;
#define GEMM_TGB_ELEMENTS \
((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \
: GEMM_K_PARTIAL_ELEMENTS) \
- : (gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
+ : (GEMM_SMALL_BATCH \
+ ? GEMM_K_PARTIAL_ELEMENTS \
+ : gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
template <
typename AT,
@@ -55,7 +63,9 @@ VARIANTS(
GemmTiling::Tile32x64x256_Simdgroups2x2,
GemmTiling::Tile64x32x256_Simdgroups4x1,
GemmTiling::Tile64x64x256_Simdgroups2x2,
- GemmTiling::Tile128x128x256_Simdgroups4x4)
+ GemmTiling::Tile128x128x256_Simdgroups4x4,
+ GemmTiling::Tile8x32x64_Simdgroups1x2x4,
+ GemmTiling::Tile8x32x128_Simdgroups1x2x4)
VARIANTS(TRANSPOSE_B, false, true)
VARIANTS(USE_MXU, false, true)
VARIANTS(
@@ -69,7 +79,8 @@ VARIANTS(GROUP_SIZE, 0, 32, 64)
VARIANTS(
A_PROLOGUE,
GemmAPrologueKind::FullPrecision,
- GemmAPrologueKind::Int8Symmetric)
+ GemmAPrologueKind::Int8Symmetric,
+ GemmAPrologueKind::PackedHalf)
VARIANTS(A_GROUP_SIZE, 0, 128)
CONSTRAINT(
USE_MXU ==
@@ -105,13 +116,21 @@ CONSTRAINT(
CONSTRAINT(
GEMM_TILING != GemmTiling::Tile16x128x256_Simdgroups1x4 ||
(TRANSPOSE_B && B_PROLOGUE == GemmBPrologueKind::FullPrecision))
-CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || USE_MXU)
+CONSTRAINT(A_PROLOGUE == GemmAPrologueKind::FullPrecision || A_PROLOGUE == GemmAPrologueKind::PackedHalf || USE_MXU)
+CONSTRAINT(GEMM_PACKED_HALF_TILING == (A_PROLOGUE == GemmAPrologueKind::PackedHalf))
+CONSTRAINT(!GEMM_PACKED_HALF_TILING || (!USE_MXU && TRANSPOSE_B && BITS == 4))
+CONSTRAINT(
+ !GEMM_SMALL_BATCH_W8 ||
+ (!USE_MXU && TRANSPOSE_B && A_PROLOGUE == GemmAPrologueKind::FullPrecision &&
+ BITS == 8))
CONSTRAINT(
A_PROLOGUE == GemmAPrologueKind::FullPrecision ||
(TRANSPOSE_B && B_PROLOGUE != GemmBPrologueKind::FullPrecision))
-CONSTRAINT((A_PROLOGUE == GemmAPrologueKind::FullPrecision) == (A_GROUP_SIZE == 0))
+CONSTRAINT((A_PROLOGUE != GemmAPrologueKind::Int8Symmetric) == (A_GROUP_SIZE == 0))
+CONSTRAINT(!A_IS_INT8 || GEMM_TILING != GemmTiling::Tile128x128x256_Simdgroups4x4)
KERNEL(Gemm)(
const device AT* a OPTIONAL(A_PROLOGUE == GemmAPrologueKind::FullPrecision),
+ const device uint* a_packed_half OPTIONAL(A_PROLOGUE == GemmAPrologueKind::PackedHalf),
const device BT* b,
device DT* d,
const device BT* scales
@@ -124,6 +143,8 @@ KERNEL(Gemm)(
OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
const device int32_t* rht_factors
OPTIONAL(output_transform.contains(GemmDTransform::RHT)),
+ const constant float& soft_cap
+ OPTIONAL(output_transform.contains(GemmDTransform::SOFT_CAP)),
const device int8_t* a_int8 OPTIONAL(A_IS_INT8),
const device float* a_scales OPTIONAL(A_IS_INT8),
const device int32_t* a_group_sums OPTIONAL(NEEDS_ASYMMETRIC_WEIGHT_CORRECTION),
@@ -159,7 +180,7 @@ KERNEL(Gemm)(
NEEDS_ASYMMETRIC_WEIGHT_CORRECTION == (A_IS_INT8 && RightOperand::NEEDS_CORRECTION),
"kernel bindings and operand correction policy must agree"
);
- const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums);
+ const auto left_storage = operands::pack_left<LeftOperand, AT>(a, a_int8, a_scales, a_group_sums, a_packed_half);
const auto right_storage =
operands::pack_right<RightOperand, BT>(b, scales, biases, zero_points, signed_codes, gemm_b_layout);
@@ -177,6 +198,20 @@ KERNEL(Gemm)(
b_shared,
thread_context
);
+ } else if constexpr (GEMM_SMALL_BATCH) {
+ using Core = SmallBatchMmaCore<DT, GEMM_TILING, LeftOperand, RightOperand>;
+ Core::run(
+ left_storage,
+ right_storage,
+ d,
+ params,
+ output_transform,
+ soft_cap,
+ output_bias,
+ rht_factors,
+ b_shared,
+ thread_context
+ );
} else {
using Core = SimdgroupMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;
Core::run(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 97c81313a..c1da24799 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -258,7 +258,7 @@ impl GemmKernel {
let use_mxu = plan.engine == GemmEngine::Mxu;
let gemm_b_layout = quantized.as_ref().map_or(GemmBLayout::RowMajor, |quantized| quantized.weights_layout);
- if gemm_b_layout == GemmBLayout::MxuTiled && (!use_mxu || shape.a_full_precision) {
+ if gemm_b_layout == GemmBLayout::MxuTiled && (!use_mxu || !shape.a_is_quantized()) {
return Err(MatmulMetalError::IncompatibleA {
path: "Gemm",
reason: "MxuTiled GEMM B layout requires int8 activations on the MXU",
@@ -282,23 +282,26 @@ impl GemmKernel {
quantized.zero_point_strides(),
),
};
- let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match &a {
+ let (a_full_precision, a_packed_half, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match &a {
MatmulA::FullPrecision {
values,
offset,
- } => (Some(values.subrange(*offset..)), None, None, None, GemmAPrologueKind::FullPrecision, None),
+ } => (Some(values.subrange(*offset..)), None, None, None, None, GemmAPrologueKind::FullPrecision, None),
+ MatmulA::PackedHalf {
+ values,
+ ..
+ } => (None, Some(values.subrange(0..)), None, None, None, GemmAPrologueKind::PackedHalf, None),
MatmulA::Int8Symmetric {
values,
scales: activation_scales,
group_sums: activation_group_sums,
- scale_group_size,
- code_layout,
+ quantization,
} => {
validate_int8_left_operand(
use_mxu,
shape,
- *scale_group_size,
- *code_layout,
+ quantization.scale_group_size(),
+ quantization.code_layout(),
activation_group_sums.is_some(),
)?;
if output_transform.contains(GemmDTransform::SOFT_CAP) {
@@ -308,12 +311,13 @@ impl GemmKernel {
});
}
(
+ None,
None,
Some(*values),
Some(*activation_scales),
*activation_group_sums,
GemmAPrologueKind::Int8Symmetric,
- Some(*scale_group_size),
+ Some(quantization.scale_group_size()),
)
},
MatmulA::Trellis {
@@ -398,6 +402,7 @@ impl GemmKernel {
let kernel = self.get_or_create(command_buffer.context(), specialization, self.output_data_type)?;
kernel.encode(
a_full_precision,
+ a_packed_half,
weights,
d.reborrow(),
scales,
@@ -405,6 +410,7 @@ impl GemmKernel {
zero_points,
output_bias,
rht_factors,
+ soft_cap,
a_int8,
a_scales,
a_group_sums,
@@ -438,20 +444,29 @@ impl GemmKernel {
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<(), MatmulMetalError> {
let (m, n, k) = (shape.m, shape.n, shape.k);
- let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {
+ let (a_full_precision, a_packed_half, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match a {
MatmulA::FullPrecision {
values,
offset,
- } => (Some(values.subrange(offset..)), None, None, None, GemmAPrologueKind::FullPrecision, None),
+ } => (Some(values.subrange(offset..)), None, None, None, None, GemmAPrologueKind::FullPrecision, None),
+ MatmulA::PackedHalf {
+ values,
+ ..
+ } => (None, Some(values.subrange(0..)), None, None, None, GemmAPrologueKind::PackedHalf, None),
MatmulA::Int8Symmetric {
values,
scales,
group_sums,
- scale_group_size,
- code_layout: _,
- } => {
- (None, Some(values), Some(scales), group_sums, GemmAPrologueKind::Int8Symmetric, Some(scale_group_size))
- },
+ quantization,
+ } => (
+ None,
+ None,
+ Some(values),
+ Some(scales),
+ group_sums,
+ GemmAPrologueKind::Int8Symmetric,
+ Some(quantization.scale_group_size()),
+ ),
MatmulA::Trellis {
..
} => {
@@ -481,7 +496,7 @@ impl GemmKernel {
)?;
let elem = (m as usize) * (n as usize);
- let partial_data_type = if a_int8.is_some() {
+ let partial_data_type = if a_int8.is_some() || tiling.is_small_batch_variant() {
DataType::F32
} else {
self.output_data_type
@@ -493,6 +508,7 @@ impl GemmKernel {
let part_kernel = self.get_or_create(command_buffer.context(), part_spec, partial_data_type)?;
part_kernel.encode(
a_full_precision,
+ a_packed_half,
weights,
&mut temp,
scales,
@@ -500,6 +516,7 @@ impl GemmKernel {
zero_points,
None::<&<Metal as Backend>::GlobalGpuBuffer>,
None::<&<Metal as Backend>::GlobalGpuBuffer>,
+ None,
a_int8,
a_scales,
a_group_sums,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
index 5ce0acd5e..7f4107a20 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -8,7 +8,7 @@ mod trellis;
pub use error::GemmSpecializationError;
pub use kernel::GemmKernel;
pub use selection::GemmPlanError;
-pub(super) use selection::GemmProblem;
+pub(super) use selection::{GemmProblem, small_batch_format, small_m_weights};
use crate::backends::metal::gpu_types::gemm::GemmTiling;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 1c4894164..0c335dc3e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -4,11 +4,17 @@ use thiserror::Error;
use crate::{
backends::{
common::{
- gpu_types::gemm::GemmBPrologueKind,
- kernel::{activation_transform::ACTIVATION_SCALE_GROUP_SIZE, matmul::MatmulShape},
+ gpu_types::{
+ gemm::{GemmBPrologueKind, GemmDTransform},
+ hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ },
+ kernel::{
+ activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
+ matmul::{ActivationFormat, MatmulShape},
+ },
},
metal::{
- gpu_types::gemm::GemmTiling,
+ gpu_types::gemm::{GemmBLayout, GemmTiling},
kernel::matmul::gemm::{GemmEngine, GemmPlan, policy},
},
},
@@ -34,6 +40,40 @@ pub enum GemmPlanError {
UnsupportedLayout(&'static str),
}
+/// The weight formats with small-M kernels (batched GEMV and register GEMM): W4 zero-point or scale-bias, and W8
+/// symmetric, at group size 32 or 64. Returns the GEMM's activation format, tiling and K alignment.
+pub fn small_m_weights(shape: &MatmulShape) -> Option<(ActivationFormat, GemmTiling, u32)> {
+ let group_size = shape.b_group_size.filter(|group_size| matches!(group_size, 32 | 64))?;
+ match (shape.b_prologue, shape.b_bits) {
+ (GemmBPrologueKind::ScaleZeroPointDequant | GemmBPrologueKind::ScaleBiasDequant, Some(4)) => Some((
+ ActivationFormat::PackedHalf {
+ group_size,
+ },
+ GemmTiling::Tile8x32x64_Simdgroups1x2x4,
+ 512,
+ )),
+ (GemmBPrologueKind::ScaleSymmetricDequant, Some(8)) => {
+ Some((ActivationFormat::Bf16, GemmTiling::Tile8x32x128_Simdgroups1x2x4, 256))
+ },
+ _ => None,
+ }
+}
+
+pub fn small_batch_format(
+ shape: MatmulShape,
+ layout: GemmBLayout,
+) -> Option<(ActivationFormat, GemmTiling)> {
+ let (activation, tiling, k_alignment) = small_m_weights(&shape)?;
+ (shape.b_transpose
+ && !shape.gathered
+ && !shape.b_is_trellis
+ && layout == GemmBLayout::RowMajor
+ && shape.b_leading_dimension.is_none()
+ && shape.k.is_multiple_of(k_alignment)
+ && (!shape.d_transform.contains(GemmDTransform::RHT) || shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE)))
+ .then_some((activation, tiling))
+}
+
impl GemmProblem {
pub fn new(
shape: MatmulShape,
@@ -63,6 +103,23 @@ impl GemmProblem {
self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family))
}
+ pub fn small_batch_plan(
+ self,
+ layout: GemmBLayout,
+ ) -> Option<GemmPlan> {
+ let (activation, tiling) = small_batch_format(self.shape, layout)?;
+ let plan = self.finish_plan(GemmEngine::Simdgroup, tiling);
+ let partition_alignment = if tiling.is_packed_w4() || plan.split_k == 1 {
+ 256
+ } else {
+ 512
+ };
+ (self.shape.a == activation
+ && (4..=8).contains(&self.shape.m)
+ && self.shape.k.is_multiple_of(partition_alignment * plan.split_k))
+ .then_some(plan)
+ }
+
#[cfg(test)]
pub(super) fn select_plan_for_engine(
self,
@@ -123,9 +180,9 @@ impl GemmProblem {
return 1;
};
let align = step.max(ACTIVATION_SCALE_GROUP_SIZE) * tiling.simdgroups_k();
- let target_tiles = policy::split_k_target_tiles(!shape.a_full_precision, tiling, shape.b_bits);
+ let target_tiles = policy::split_k_target_tiles(shape.a_is_quantized(), tiling, shape.b_bits);
let mut split_k = (target_tiles / base_tiles).max(1).min((shape.k / align).max(1));
- if !shape.a_full_precision && engine == GemmEngine::Mxu && tiling.block_k() != 0 {
+ if shape.a_is_quantized() && engine == GemmEngine::Mxu && tiling.block_k() != 0 {
split_k = split_k.min((shape.k / tiling.block_k()).max(1));
}
while split_k > 1 && !shape.k.is_multiple_of(split_k * align) {
@@ -198,12 +255,12 @@ pub(super) fn outer_block_k(
}
fn mxu_is_eligible(shape: MatmulShape) -> bool {
- if !shape.a_full_precision || shape.b_prologue == GemmBPrologueKind::FullPrecision {
+ if shape.a_is_quantized() || shape.b_prologue == GemmBPrologueKind::FullPrecision {
return true;
}
shape.b_transpose
&& shape.b_leading_dimension.is_none()
- && shape.k.is_multiple_of(policy::mxu_mn_tile(!shape.a_full_precision, shape.m, shape.n).block_k())
+ && shape.k.is_multiple_of(policy::mxu_mn_tile(shape.a_is_quantized(), shape.m, shape.n).block_k())
}
fn select_tiling(
@@ -214,7 +271,7 @@ fn select_tiling(
match engine {
GemmEngine::Simdgroup if shape.is_quant() => policy::simdgroup_quant_tile(shape.m, shape.n, apple_gpu_family),
GemmEngine::Simdgroup => policy::simdgroup_fp_tile(shape.m, shape.n, shape.k),
- GemmEngine::Mxu if !shape.a_full_precision => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {
+ GemmEngine::Mxu if shape.a_is_quantized() => match policy::mxu_int8_gemm_tile(shape.m, shape.n) {
tiling
if tiling.simdgroups_k() > 1
&& !shape.k.is_multiple_of(tiling.simdgroups_k() * ACTIVATION_SCALE_GROUP_SIZE) =>
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 1b1c4b7f6..6201cb429 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -61,6 +61,7 @@ impl GemmSpecialization {
let valid_activation_group = match self.a_prologue {
GemmAPrologueKind::FullPrecision => self.a_group_size.is_none(),
GemmAPrologueKind::Int8Symmetric => self.a_group_size == Some(ACTIVATION_SCALE_GROUP_SIZE),
+ GemmAPrologueKind::PackedHalf => self.a_group_size.is_none(),
};
if !valid_activation_group {
return Err(GemmSpecializationError::InvalidAGroupSize {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index b374e8727..041cf9a00 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -12,6 +12,19 @@ using namespace metal;
using namespace uzu;
using namespace uzu::gemm;
+#define GEMV_BATCHED_INPUT (INPUT_ALIGNED && GROUP_LANES == 1)
+#define GEMV_FULL_PRECISION_TILE \
+ (BITS == 0 && NUM_SIMDGROUPS == 8 && REDUCTION_LANES == 32 && GROUP_LANES == 1 && (INPUT_ALIGNED || K_SPLIT == 1) && \
+ (OUTPUT_ROW_TILE == 8 / K_SPLIT || OUTPUT_ROW_TILE == 32 / K_SPLIT) && INPUT_ROW_TILE == 1)
+#define GEMV_QUANTIZED_ROW_TILE \
+ (K_SPLIT == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE == 1 && \
+ ((BITS == 4 && GROUP_SIZE == 16 * GROUP_LANES) || (BITS == 8 && GROUP_SIZE == 8 * GROUP_LANES)))
+#define GEMV_BATCHED_FMA_32_TILE \
+ (K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 4 && OUTPUT_ROW_TILE == 32 && INPUT_ROW_TILE >= 2 && \
+ INPUT_ROW_TILE <= 4)
+#define GEMV_BATCHED_FMA_16_TILE \
+ (K_SPLIT == 1 && REDUCTION_LANES == 8 && NUM_SIMDGROUPS == 2 && OUTPUT_ROW_TILE == 16 && INPUT_ROW_TILE >= 2 && \
+ INPUT_ROW_TILE <= 4)
template <
typename AT,
typename BT,
@@ -41,66 +54,22 @@ VARIANTS(GROUP_SIZE, 0, 32, 64)
VARIANTS(BITS, 0, 4, 8)
VARIANTS(K_SPLIT, 1, 2, 4, 8)
VARIANTS(INPUT_ALIGNED, false, true)
-VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4, 5, 6, 7, 8)
-VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32, 64)
-VARIANTS(REDUCTION_LANES, 8, 16, 32)
-VARIANTS(GROUP_LANES, 1, 2, 4, 8, 16)
+VARIANTS(INPUT_ROW_TILE, 1, 2, 3, 4)
+VARIANTS(OUTPUT_ROW_TILE, 1, 2, 4, 8, 16, 32)
+VARIANTS(REDUCTION_LANES, 8, 32)
+VARIANTS(GROUP_LANES, 1, 2, 4, 8)
VARIANTS(NUM_SIMDGROUPS, 2, 4, 8)
-CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
-CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)
-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
-CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)
-CONSTRAINT(BITS == 0 || K_SPLIT == 1)
-CONSTRAINT(BITS != 0 || (INPUT_ROW_TILE == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && GROUP_LANES == 1))
-CONSTRAINT(INPUT_ROW_TILE != 1 || REDUCTION_LANES == 32)
CONSTRAINT(
- INPUT_ROW_TILE == 1 ||
- (K_SPLIT == 1 && INPUT_ALIGNED && AT == "bfloat" && DT == "bfloat" && GROUP_LANES == 1))
-
-#define GEMV_TILE(OUTPUT_ROWS, LANES, SIMD_GROUPS) \
- (OUTPUT_ROW_TILE == OUTPUT_ROWS && REDUCTION_LANES == LANES && NUM_SIMDGROUPS == SIMD_GROUPS)
-#define INPUT_ROWS(FIRST, LAST) (INPUT_ROW_TILE >= FIRST && INPUT_ROW_TILE <= LAST)
-
-CONSTRAINT(
- INPUT_ROW_TILE == 1 ||
- (B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant && GROUP_SIZE == 64 &&
- ((GEMV_TILE(16, 8, 2) && INPUT_ROWS(2, 7)) ||
- (BITS == 4 && GEMV_TILE(16, 16, 4) && INPUT_ROWS(3, 6)) ||
- (BITS == 4 && GEMV_TILE(8, 8, 2) && INPUT_ROW_TILE == 2))))
-
-// Keep only selector-reachable geometry families.
-CONSTRAINT(
- BITS == 0 ||
- (NUM_SIMDGROUPS == 2 &&
- (OUTPUT_ROW_TILE == 2 || OUTPUT_ROW_TILE == 4 || OUTPUT_ROW_TILE == 8 ||
- (INPUT_ROW_TILE > 1 && OUTPUT_ROW_TILE == 16))) ||
- (NUM_SIMDGROUPS == 4 && (OUTPUT_ROW_TILE == 8 || OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)) ||
- (NUM_SIMDGROUPS == 8 && (OUTPUT_ROW_TILE == 16 || OUTPUT_ROW_TILE == 32)))
-CONSTRAINT(
- BITS == 0 || (AT == "bfloat" && DT == "bfloat") ||
- (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32))
-CONSTRAINT(BITS != 8 || (NUM_SIMDGROUPS == 8 && OUTPUT_ROW_TILE == 32) || INPUT_ROW_TILE > 1)
-CONSTRAINT(INPUT_ALIGNED || K_SPLIT == 1)
-CONSTRAINT(K_SPLIT <= NUM_SIMDGROUPS && NUM_SIMDGROUPS % K_SPLIT == 0)
-CONSTRAINT(OUTPUT_ROW_TILE % (NUM_SIMDGROUPS / K_SPLIT) == 0)
-CONSTRAINT(OUTPUT_ROW_TILE / (NUM_SIMDGROUPS / K_SPLIT) % (32 / REDUCTION_LANES) == 0)
-CONSTRAINT(
- (BITS == 0 && (OUTPUT_ROW_TILE == (NUM_SIMDGROUPS / K_SPLIT) ||
- OUTPUT_ROW_TILE == 4 * (NUM_SIMDGROUPS / K_SPLIT))) ||
- (BITS != 0 &&
- (OUTPUT_ROW_TILE == NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 2 * NUM_SIMDGROUPS ||
- OUTPUT_ROW_TILE == 4 * NUM_SIMDGROUPS || OUTPUT_ROW_TILE == 8 * NUM_SIMDGROUPS)))
-CONSTRAINT(REDUCTION_LANES % GROUP_LANES == 0)
-CONSTRAINT(BITS == 0 || GROUP_SIZE % GROUP_LANES == 0)
-CONSTRAINT(BITS != 4 || (GROUP_SIZE / GROUP_LANES) % 16 == 0)
-CONSTRAINT(BITS != 8 || (GROUP_SIZE / GROUP_LANES) % 8 == 0)
-CONSTRAINT(BITS != 0 || REDUCTION_LANES == 32)
+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision && BITS == 0 && GROUP_SIZE == 0) ||
+ ((B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant ||
+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) &&
+ BITS == 4 && GROUP_SIZE != 0) ||
+ (B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant && BITS == 8 && GROUP_SIZE != 0))
+CONSTRAINT(INPUT_ROW_TILE == 1 || GEMV_BATCHED_INPUT)
CONSTRAINT(
- BITS == 0 || INPUT_ROW_TILE > 1 ||
- (BITS == 4 && ((GROUP_SIZE == 32 && GROUP_LANES == 2) || (GROUP_SIZE == 64 && GROUP_LANES == 4))) ||
- (BITS == 8 && ((GROUP_SIZE == 32 && GROUP_LANES == 4) || (GROUP_SIZE == 64 && GROUP_LANES == 8))))
+ GEMV_FULL_PRECISION_TILE ||
+ (BITS != 0 && (GEMV_QUANTIZED_ROW_TILE || GEMV_BATCHED_FMA_32_TILE || GEMV_BATCHED_FMA_16_TILE)))
KERNEL(Gemv)(
const device uint32_t* b,
const device BT* scales
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 0a29f270d..7c2f56b33 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -1,5 +1,3 @@
-use metal::MTLGPUFamily;
-
use crate::{
backends::{
common::{
@@ -58,7 +56,6 @@ impl GemvSpecialization {
input_data_type: DataType,
output_data_type: DataType,
gpu_core_count: u32,
- apple_gpu_family: MTLGPUFamily,
) -> Option<Self> {
let is_quant = shape.is_quant();
let bits = shape.b_bits.unwrap_or(0);
@@ -74,51 +71,26 @@ impl GemvSpecialization {
let tile = if is_quant && shape.gathered {
policy::gathered_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n)
} else if is_quant {
- policy::quantized_tile(
- gpu_core_count,
- apple_gpu_family,
- bits,
- shape.b_group_size.unwrap_or(0),
- shape.m,
- shape.n,
- shape.k,
- bf16_io,
- )
- .map(|tile| match gemm_b_layout {
- GemmBLayout::MxuTiled if mxu_tiled_kernel => {
- policy::GemvTile::quantized_output_tile(8, 32, 1, 32, tile.group_lanes)
- },
- GemmBLayout::MxuTiled if tile.output_row_tile() < 16 => {
- policy::GemvTile::quantized_output_tile(8, 16, 1, 32, tile.group_lanes)
- },
- _ => tile,
- })
+ policy::quantized_tile(bits, shape.b_group_size.unwrap_or(0), shape.m, shape.n, bf16_io)
} else {
- let mixed_precision = weights_data_type == DataType::F32
- && (input_data_type != DataType::F32 || output_data_type != DataType::F32);
- if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {
+ if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {
return None;
}
let input_aligned = shape.k.is_multiple_of(FP_K_BLOCK);
- Some(policy::fp_tile(gpu_core_count, apple_gpu_family, shape.m, shape.n, shape.k, input_aligned))
+ Some(policy::fp_tile(shape.m, shape.n, shape.k, input_aligned))
};
- Self::select_tile(shape, gemm_b_layout, weights_data_type, input_data_type, output_data_type, tile?).map(
- |specialization| Self {
- mxu_tiled_kernel,
- ..specialization
- },
- )
+ Self::select_tile(shape, gemm_b_layout, tile?).map(|specialization| Self {
+ mxu_tiled_kernel,
+ ..specialization
+ })
}
pub fn select_tile(
shape: &MatmulShape,
gemm_b_layout: GemmBLayout,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
tile: policy::GemvTile,
) -> Option<Self> {
- if !shape.b_transpose || !shape.a_full_precision {
+ if !shape.b_transpose || shape.a != crate::backends::common::kernel::matmul::ActivationFormat::Bf16 {
return None;
}
let is_quant = shape.is_quant();
@@ -130,14 +102,9 @@ impl GemvSpecialization {
if bad_leading_dimension {
return None;
}
- let output_transform = output_transform_for_tile(shape, tile)?;
let bits = shape.b_bits.unwrap_or(0);
- if !is_quant {
- let mixed_precision = weights_data_type == DataType::F32
- && (input_data_type != DataType::F32 || output_data_type != DataType::F32);
- if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH {
- return None;
- }
+ if !is_quant && (shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > GEMV_MAX_BATCH) {
+ return None;
}
let block_size = if !is_quant {
FP_K_BLOCK
@@ -147,10 +114,11 @@ impl GemvSpecialization {
256
};
let input_aligned = shape.k.is_multiple_of(block_size);
- // Gathered quantized rows cannot share one input tile.
- if is_quant && shape.gathered && tile.input_row_tile > 1 {
+ // Gathered quantized rows cannot share one input tile, and batched tiles need whole K blocks.
+ if is_quant && tile.input_row_tile > 1 && (shape.gathered || !input_aligned) {
return None;
}
+ let output_transform = output_transform_for_tile(shape, tile)?;
let specialization = Self {
b_prologue: shape.b_prologue,
group_size: shape.b_group_size.unwrap_or(0),
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
index cc988e389..ce64b7eb8 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy.rs
@@ -1,7 +1,3 @@
-use metal::MTLGPUFamily;
-
-use crate::backends::metal::context::LARGE_MIN_GPU_CORES;
-
mod quantized;
pub(super) use quantized::{gathered_tile, select as quantized_tile};
@@ -28,10 +24,6 @@ pub struct GemvTile {
pub(super) group_lanes: u32,
}
-const SMALL_G13_HUGE_N: u32 = 32768;
-const SMALL_G13_BROAD_ROW_N: u32 = 6144;
-const DEEP_K: u32 = 8192;
-const FP_LARGE_SPLIT_K_MIN_DEPTH: u32 = 4 * FP_K_BLOCK;
const FP_K_DEPTH_N_MAX: u32 = 4095;
const FP_K_DEPTH_DEEP_MIN: u32 = 3072;
const FP_K_DEPTH_VERY_DEEP_RATIO: u32 = 16;
@@ -142,29 +134,18 @@ fn preferred_fp_k_split(
/// Selects the full-precision GEMV tile. `m` is the input-vector count,
/// `n` is the output row count, and `k` is the reduction depth.
pub(super) fn fp_tile(
- gpu_core_count: u32,
- apple_gpu_family: MTLGPUFamily,
m: u32,
n: u32,
k: u32,
input_aligned: bool,
) -> GemvTile {
- let is_large_gpu = gpu_core_count >= LARGE_MIN_GPU_CORES;
- let is_small_legacy = !is_large_gpu && apple_gpu_family < MTLGPUFamily::Apple8;
- // SG8 is the portable full-precision geometry.
- let should_disable_k_split = !input_aligned
- || (m == 1 && is_large_gpu && k < FP_LARGE_SPLIT_K_MIN_DEPTH)
- || (m == 1 && is_small_legacy && n >= SMALL_G13_HUGE_N);
-
- let k_split = if should_disable_k_split {
- 1
- } else {
+ // SG8 is the portable full-precision geometry; one output row per SIMD group for a single vector.
+ let k_split = if input_aligned {
cap_k_split_to_complete_fp_k_blocks(k, preferred_fp_k_split(m, n, k))
+ } else {
+ 1
};
-
- let results_per_simdgroup = if is_small_legacy && m == 1 && n >= SMALL_G13_BROAD_ROW_N {
- DEFAULT_RESULTS_PER_SIMDGROUP
- } else if m == 1 && (k <= DEEP_K || !is_large_gpu) {
+ let results_per_simdgroup = if m == 1 {
1
} else {
DEFAULT_RESULTS_PER_SIMDGROUP
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
index 43dd010ff..035b8a042 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/policy/quantized.rs
@@ -1,10 +1,4 @@
-use metal::MTLGPUFamily;
-
use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile};
-use crate::backends::metal::context::LARGE_MIN_GPU_CORES;
-
-const QUANT_N_BUCKET_MAXES: [u32; 6] = [512, 2048, 4096, 8192, 16384, 32768];
-const QUANT_K_BUCKET_MAXES: [u32; 3] = [512, 2048, 8192];
/// Lane-sliced tiles split each quantization group across reduction lanes.
/// A lane stages 8 bytes: 16 W4 values or 8 W8 values.
@@ -42,86 +36,16 @@ pub fn gathered_tile(
(n >= tile.rows_per_lane()).then_some(tile)
}
-fn table_bucket_index(
- value: u32,
- bucket_maxes: &[u32],
-) -> usize {
- bucket_maxes.partition_point(|&max| value > max)
-}
-
-fn lane_policy(
- gpu_core_count: u32,
- apple_gpu_family: MTLGPUFamily,
- m: u32,
- n: u32,
- k: u32,
- bits: u32,
- group: u32,
-) -> GemvTile {
- let is_large_gpu = gpu_core_count >= LARGE_MIN_GPU_CORES;
- if m != 1 || bits != 4 {
- return lane_default(bits, group);
- }
-
- let k_bucket = table_bucket_index(k, &QUANT_K_BUCKET_MAXES);
- let n_bucket = table_bucket_index(n, &QUANT_N_BUCKET_MAXES);
- let (num_simdgroups, results_per_simdgroup) = match (is_large_gpu, apple_gpu_family, k_bucket, n_bucket) {
- (true, _, 0, 1) => (4, 2),
- (true, _, 1, 0) => (2, 1),
- (true, _, 1, 1..=3) => (2, 2),
- (true, _, 1, 4) => (2, 1),
- (true, _, 1, 5) => (2, 4),
- (true, _, 2, 1) => (4, 2),
- (true, _, 3, 1) => (2, 2),
- (false, MTLGPUFamily::Apple9, 0, 1) => (4, 4),
- (false, MTLGPUFamily::Apple9, 1, 0) => (4, 2),
- (false, MTLGPUFamily::Apple9, 1, 1) => (2, 2),
- (false, MTLGPUFamily::Apple9, 1, 2) => (4, 2),
- (false, MTLGPUFamily::Apple9, 1, 4) => (2, 2),
- (false, MTLGPUFamily::Apple9, 1, 5) => (4, 2),
- (false, MTLGPUFamily::Apple9, 2, 1) => (4, 2),
- (false, MTLGPUFamily::Apple9, 3, 1) => (2, 2),
- (false, MTLGPUFamily::Apple8, 0, 1) => (4, 4),
- (false, MTLGPUFamily::Apple8, 1, _) | (false, MTLGPUFamily::Apple8, 2, 1) => (8, 2),
- (false, family, 0, 1) if family < MTLGPUFamily::Apple8 => (4, 8),
- (false, family, 1, 0..=3) if family < MTLGPUFamily::Apple8 => (8, 2),
- _ => (8, 4),
- };
- let selected = lane_tile(num_simdgroups, results_per_simdgroup, bits, group);
- if n < selected.results_per_simdgroup {
- lane_default(bits, group)
- } else {
- selected
- }
-}
-
pub fn select(
- gpu_core_count: u32,
- apple_gpu_family: MTLGPUFamily,
bits: u32,
group: u32,
m: u32,
n: u32,
- k: u32,
bf16_io: bool,
) -> Option<GemvTile> {
- if !matches!(bits, 4 | 8) || !matches!(group, 32 | 64) || !(1..=8).contains(&m) {
- return None;
- }
-
- if !bf16_io {
- let tile = lane_default(bits, group);
- return (m <= 4 && n >= tile.rows_per_lane()).then_some(tile);
- }
-
- let tile = match m {
- 1..=4 => lane_policy(gpu_core_count, apple_gpu_family, m, n, k, bits, group),
- 5..=8 if n <= 64 => lane_default(bits, group),
- _ => return None,
- };
- // Input rows are tiled independently; partial tiles clamp loads and stores.
- if n < tile.rows_per_lane() {
+ // The GEMV pipelines exist for BF16 I/O only. Rows 5..=8 stay on GEMV for tiny N only.
+ if !bf16_io || (m > 4 && n > 64) {
return None;
}
- Some(tile)
+ gathered_tile(bits, group, m, n)
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index dc5becb57..0f8cb2afe 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -20,11 +20,10 @@ use crate::{
gpu_types::gemm::{GemmBLayout, GemmTiling},
kernel::matmul::{
GemmKernel, MatmulMetalError,
- gemm::{self, GemmPlan, GemmProblem},
- gemv::{GemvKernel, GemvSpecialization},
+ gemm::{self, GemmPlan, GemmProblem, small_batch_format, small_m_weights},
+ gemv::{GemvKernel, GemvSpecialization, GemvTile},
matmul_dispatch::MatmulDispatch,
matmul_output_work::MatmulOutputWork,
- qmv::{self, QmvRoute},
},
},
},
@@ -84,7 +83,6 @@ impl MatmulMetalKernel {
pub fn choose_dispatch(
shape: &MatmulShape,
gemm_b_layout: GemmBLayout,
- device_name: &str,
gpu_core_count: u32,
apple_gpu_family: MTLGPUFamily,
supports_mxu: bool,
@@ -92,28 +90,23 @@ impl MatmulMetalKernel {
input_data_type: DataType,
output_data_type: DataType,
) -> MatmulDispatch {
- let all_bf16 = weights_data_type == DataType::BF16
- && input_data_type == DataType::BF16
- && output_data_type == DataType::BF16;
let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);
if shape.b_is_trellis {
return MatmulDispatch::Gemm(problem.select_plan());
}
- if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {
- return match route {
- QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(
- GemvSpecialization::select_tile(
- shape,
- gemm_b_layout,
- weights_data_type,
- input_data_type,
- output_data_type,
- tile,
- )
- .expect("typed QMV route must contain a legal GEMV tile"),
- ),
- QmvRoute::MainGemm(plan) => MatmulDispatch::Gemm(plan),
+ // Small M, one rule for every chip: batched GEMV for M=2..3 (M=2..4 with MXU), the register GEMM above that.
+ if all_bf16(weights_data_type, input_data_type, output_data_type) && small_m_weights(shape).is_some() {
+ if !supports_mxu && let Some(plan) = problem.small_batch_plan(gemm_b_layout) {
+ return MatmulDispatch::Gemm(plan);
+ }
+ let tile = match (supports_mxu, shape.m) {
+ (false, 2..=8) => Some(GemvTile::quantized_output_tile(4, 32, shape.m.min(4), 8, 1)),
+ (true, 2..=4) => Some(GemvTile::quantized_output_tile(2, 16, shape.m, 8, 1)),
+ _ => None,
};
+ if let Some(gemv) = tile.and_then(|tile| GemvSpecialization::select_tile(shape, gemm_b_layout, tile)) {
+ return MatmulDispatch::Gemv(gemv);
+ }
}
let gemv = GemvSpecialization::select_shape(
shape,
@@ -122,7 +115,6 @@ impl MatmulMetalKernel {
input_data_type,
output_data_type,
gpu_core_count,
- apple_gpu_family,
);
let plan = problem.select_plan();
match gemv {
@@ -145,7 +137,6 @@ impl MatmulMetalKernel {
Self::choose_dispatch(
shape,
gemm_b_layout,
- &context.device_name,
context.gpu_core_count,
context.apple_gpu_family,
context.supports_mxu,
@@ -156,6 +147,14 @@ impl MatmulMetalKernel {
}
}
+fn all_bf16(
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+) -> bool {
+ (weights_data_type, input_data_type, output_data_type) == (DataType::BF16, DataType::BF16, DataType::BF16)
+}
+
impl MatmulKernel for MatmulMetalKernel {
type Backend = Metal;
type Error = MatmulMetalError;
@@ -200,7 +199,6 @@ impl MatmulKernel for MatmulMetalKernel {
if !context.supports_mxu
|| self.input_data_type != DataType::BF16
|| self.output_data_type != DataType::BF16
- || shape.a_full_precision
|| !shape.is_quant()
|| shape.params_layout != Some(QuantParamsLayout::GroupOutput)
|| !supports_integer_right_operand(shape)
@@ -217,29 +215,52 @@ impl MatmulKernel for MatmulMetalKernel {
Some(quantization)
}
- fn select_activation_format(
+ /// Load-time: the activation format this linear prepares once (Int8 on MXU, PackedHalf for the small-batch W4
+ /// GEMM, else Bf16). `select_activation_format` decides per batch size whether it is used.
+ fn activation_preparation(
&self,
- bf16_shape: &MatmulShape,
+ shape: &MatmulShape,
weights_layout: GemmBLayout,
context: &MetalContext,
) -> ActivationFormat {
- if bf16_shape.params_layout != Some(QuantParamsLayout::GroupOutput)
- || !supports_integer_right_operand(bf16_shape)
- || matches!(self.select_dispatch(bf16_shape, weights_layout, context), MatmulDispatch::Gemv(_))
- {
- return ActivationFormat::Bf16;
+ if let Some(quantization) = self.select_activation_quantization(shape, context) {
+ return ActivationFormat::Int8(quantization);
}
- if weights_layout == GemmBLayout::MxuTiled {
- return ActivationFormat::Int8;
+ match small_batch_format(*shape, weights_layout) {
+ Some((format, _))
+ if !context.supports_mxu
+ && shape.params_layout == Some(QuantParamsLayout::GroupOutput)
+ && all_bf16(self.weights_data_type, self.input_data_type, self.output_data_type) =>
+ {
+ format
+ },
+ _ => ActivationFormat::Bf16,
}
+ }
- let a8_shape = MatmulShape {
- a_full_precision: false,
- ..*bf16_shape
+ /// Encode-time: `prepared` when this batch size has a kernel for it, else Bf16 (GEMV keeps Bf16 input).
+ fn select_activation_format(
+ &self,
+ bf16_shape: &MatmulShape,
+ weights_layout: GemmBLayout,
+ prepared: ActivationFormat,
+ context: &MetalContext,
+ ) -> ActivationFormat {
+ let dispatch = |shape: &MatmulShape| self.select_dispatch(shape, weights_layout, context);
+ let usable = match prepared {
+ ActivationFormat::PackedHalf {
+ ..
+ } => matches!(
+ dispatch(&MatmulShape { a: prepared, ..*bf16_shape }),
+ MatmulDispatch::Gemm(plan) if plan.tiling.is_small_batch_variant()
+ ),
+ ActivationFormat::Int8(_) => matches!(dispatch(bf16_shape), MatmulDispatch::Gemm(_)),
+ _ => false,
};
- match self.select_dispatch(&a8_shape, weights_layout, context) {
- MatmulDispatch::Gemm(plan) if plan.engine == gemm::GemmEngine::Mxu => ActivationFormat::Int8,
- _ => ActivationFormat::Bf16,
+ if usable {
+ prepared
+ } else {
+ ActivationFormat::Bf16
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index 1a5803108..58e6b5ce8 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -4,12 +4,9 @@ mod gemv;
mod matmul_dispatch;
mod matmul_metal_kernel;
mod matmul_output_work;
-mod qmv;
pub use error::MatmulMetalError;
pub use gemm::GemmKernel;
-#[cfg(test)]
-use matmul_dispatch::MatmulDispatch;
pub use matmul_metal_kernel::MatmulMetalKernel;
use matmul_metal_kernel::supports_integer_right_operand;
pub use matmul_output_work::MatmulOutputWork;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs
deleted file mode 100644
index b75f223f9..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/mod.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-mod routes;
-
-pub(super) use routes::route;
-
-use super::{gemm::GemmPlan, gemv::GemvTile};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(super) enum QmvRoute {
- Tuned(GemvTile),
- MainGemv(GemvTile),
- MainGemm(GemmPlan),
-}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
deleted file mode 100644
index 2042c758a..000000000
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/qmv/routes.rs
+++ /dev/null
@@ -1,239 +0,0 @@
-// Qwen3.6-27B W4/ZP and W8/Symmetric, G64, plus Weaver linears on M5 Max.
-// Tuned on M1, M2, M2 Pro, M3 Max, M4, M4 Pro, M5 Max.
-use metal::MTLGPUFamily;
-
-use super::{
- super::{
- gemm::{GemmEngine, GemmPlan},
- gemv::GemvTile,
- },
- QmvRoute,
-};
-use crate::backends::{
- common::{gpu_types::gemm::GemmBPrologueKind, kernel::matmul::MatmulShape},
- metal::gpu_types::gemm::GemmTiling,
-};
-
-const DOWN: u8 = 1 << 0;
-const GATE: u8 = 1 << 1;
-const GATE_UP: u8 = 1 << 2;
-const PROJECTION_IN: u8 = 1 << 3;
-const PROJECTION_OUT: u8 = 1 << 4;
-const READOUT: u8 = 1 << 5;
-const WEAVER: u8 = 1 << 6;
-
-#[derive(Clone, Copy)]
-struct RouteRow {
- device_name: &'static str,
- apple_gpu_family: MTLGPUFamily,
- bits: u32,
- group: u32,
- m: u32,
- shapes: u8,
- route: QmvRoute,
-}
-
-const fn shape(
- n: u32,
- k: u32,
-) -> u8 {
- match (n, k) {
- (5120, 17408) => DOWN,
- (6144, 5120) => GATE,
- (34816, 5120) => GATE_UP,
- (16480, 5120) => PROJECTION_IN,
- (5120, 6144) => PROJECTION_OUT,
- (248320, 5120) => READOUT,
- (5120 | 6144, 2048) => WEAVER,
- _ => 0,
- }
-}
-
-#[rustfmt::skip]
-macro_rules! tuned { ($input:literal, $output:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::Tuned(GemvTile::quantized_output_tile($simdgroups, $output, $input, $lanes, $group_lanes)) }; }
-#[rustfmt::skip]
-macro_rules! main_gemv { ($input:literal, $results_per_simdgroup:literal, $lanes:literal, $group_lanes:literal, $simdgroups:literal) => { QmvRoute::MainGemv(GemvTile::quantized($simdgroups, $results_per_simdgroup, $input, $lanes, $group_lanes)) }; }
-#[rustfmt::skip]
-macro_rules! main_gemm { ($engine:ident, $tiling:ident, $split:literal) => { QmvRoute::MainGemm(GemmPlan { engine: GemmEngine::$engine, tiling: GemmTiling::$tiling, split_k: $split }) }; }
-#[rustfmt::skip]
-macro_rules! qmv_format { (w4_zp_g64) => { (4, 64) }; (w8_sym_g64) => { (8, 64) }; }
-#[rustfmt::skip]
-macro_rules! measured_device_name { (M1) => { "Apple M1" }; (M2) => { "Apple M2" }; (M2Pro) => { "Apple M2 Pro" }; (M3Max) => { "Apple M3 Max" }; (M4) => { "Apple M4" }; (M4Pro) => { "Apple M4 Pro" }; (M5Max) => { "Apple M5 Max" }; }
-#[rustfmt::skip]
-macro_rules! measured_apple_gpu_family { (M1) => { MTLGPUFamily::Apple7 }; (M2) => { MTLGPUFamily::Apple8 }; (M2Pro) => { MTLGPUFamily::Apple8 }; (M3Max) => { MTLGPUFamily::Apple9 }; (M4) => { MTLGPUFamily::Apple9 }; (M4Pro) => { MTLGPUFamily::Apple9 }; (M5Max) => { MTLGPUFamily::Apple10 }; }
-#[rustfmt::skip]
-macro_rules! row { ($device:ident, $format:ident, $m:literal, $shapes:expr, $route:expr) => { RouteRow { device_name: measured_device_name!($device), apple_gpu_family: measured_apple_gpu_family!($device), bits: qmv_format!($format).0, group: qmv_format!($format).1, m: $m, shapes: $shapes, route: $route } }; }
-
-#[rustfmt::skip]
-const ROWS: &[RouteRow] = &[
- row!(M1, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M1, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 16, 1, 4)),
- row!(M1, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M1, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M1, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M1, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 2, PROJECTION_IN, tuned!(2, 8, 8, 1, 2)),
- row!(M2, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 2, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M2, w8_sym_g64, 2, DOWN, tuned!(2, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M2, w8_sym_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 3, PROJECTION_OUT, tuned!(3, 16, 16, 1, 4)),
- row!(M2Pro, w4_zp_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),
- row!(M2Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 16, 1, 4)),
- row!(M2Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M2Pro, w8_sym_g64, 3, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M2Pro, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 4, DOWN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M2Pro, w8_sym_g64, 4, GATE_UP | PROJECTION_IN | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M2Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 2, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M3Max, w4_zp_g64, 2, DOWN | GATE | PROJECTION_OUT, tuned!(2, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 3, DOWN | GATE | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 3, GATE_UP | PROJECTION_IN | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g64, 4, DOWN | GATE | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 4, GATE_UP | PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g64, 5, DOWN | GATE | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 5, GATE_UP, tuned!(5, 16, 16, 1, 4)),
- row!(M3Max, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M3Max, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M3Max, w8_sym_g64, 3, GATE_UP | PROJECTION_IN | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M3Max, w8_sym_g64, 3, DOWN | PROJECTION_OUT, tuned!(3, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 4, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),
- row!(M3Max, w8_sym_g64, 4, DOWN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M3Max, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4, w4_zp_g64, 3, DOWN, tuned!(3, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 3, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M4, w4_zp_g64, 4, DOWN | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 4, GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT, tuned!(4, 16, 16, 1, 4)),
- row!(M4, w4_zp_g64, 5, DOWN | GATE_UP | PROJECTION_IN | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 5, GATE | PROJECTION_OUT, tuned!(5, 16, 16, 1, 4)),
- row!(M4, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 4, 8)),
- row!(M4Pro, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 4, PROJECTION_IN, tuned!(4, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 5, PROJECTION_IN, tuned!(5, 16, 16, 1, 4)),
- row!(M4Pro, w4_zp_g64, 6, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4Pro, w4_zp_g64, 7, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 2, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4Pro, w8_sym_g64, 2, READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, main_gemv!(1, 4, 32, 8, 8)),
- row!(M4Pro, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 6, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M4Pro, w8_sym_g64, 7, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(7, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 2, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 3, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 4, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT | WEAVER, tuned!(4, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 5, DOWN | GATE | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M5Max, w4_zp_g64, 6, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w4_zp_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w4_zp_g64, 6, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
- row!(M5Max, w4_zp_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
- row!(M5Max, w4_zp_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w4_zp_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w4_zp_g64, 7, GATE, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 5)),
- row!(M5Max, w4_zp_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
- row!(M5Max, w8_sym_g64, 2, GATE_UP | PROJECTION_IN, main_gemv!(1, 4, 32, 8, 8)),
- row!(M5Max, w8_sym_g64, 2, DOWN | PROJECTION_OUT | READOUT, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 3, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(3, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 4, PROJECTION_IN, tuned!(2, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 4, DOWN | GATE_UP | PROJECTION_OUT | READOUT, tuned!(4, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 5, DOWN | GATE_UP | PROJECTION_IN | PROJECTION_OUT | READOUT, tuned!(5, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 6, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w8_sym_g64, 6, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
- row!(M5Max, w8_sym_g64, 6, GATE_UP | PROJECTION_IN | READOUT, tuned!(6, 16, 8, 1, 2)),
- row!(M5Max, w8_sym_g64, 7, GATE_UP | PROJECTION_IN | READOUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 1)),
- row!(M5Max, w8_sym_g64, 7, DOWN, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 4)),
- row!(M5Max, w8_sym_g64, 7, PROJECTION_OUT, main_gemm!(Mxu, Tile32x64x256_Simdgroups2x2, 6)),
-];
-
-fn qmv_format(shape: &MatmulShape) -> Option<(u32, u32)> {
- match (shape.b_prologue, shape.b_bits, shape.b_group_size) {
- (GemmBPrologueKind::ScaleZeroPointDequant, Some(4), Some(64)) => Some((4, 64)),
- (GemmBPrologueKind::ScaleSymmetricDequant, Some(8), Some(64)) => Some((8, 64)),
- _ => None,
- }
-}
-
-pub fn route(
- device_name: &str,
- apple_gpu_family: MTLGPUFamily,
- supports_mxu: bool,
- shape: &MatmulShape,
- all_bf16: bool,
-) -> Option<QmvRoute> {
- if !all_bf16
- || shape.m < 2
- || shape.m > 7
- || !shape.a_full_precision
- || !shape.b_transpose
- || shape.b_leading_dimension.is_some()
- || shape.gathered
- {
- return None;
- }
- let (bits, group) = qmv_format(shape)?;
- let mask = self::shape(shape.n, shape.k);
- if mask == 0 {
- return None;
- }
- let matches =
- |row: &&RouteRow| row.bits == bits && row.group == group && row.m == shape.m && row.shapes & mask != 0;
- let route =
- ROWS.iter().filter(matches).find(|row| row.device_name == device_name).map(|row| row.route).or_else(|| {
- let same_family = |row: &&RouteRow| row.apple_gpu_family == apple_gpu_family;
- let mut routes = ROWS.iter().filter(matches).filter(same_family).map(|row| row.route);
- let route = routes.next()?;
- routes.all(|candidate| candidate == route).then_some(route)
- });
- route.filter(|route| {
- !matches!(
- route,
- QmvRoute::MainGemm(GemmPlan {
- engine: GemmEngine::Mxu,
- ..
- })
- ) || supports_mxu
- })
-}
-
-#[cfg(test)]
-#[path = "../../../../../../unit/backends/metal/kernel/matmul/qmv/routes_test.rs"]
-mod tests;
diff --git a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang
index 70d933335..f1f147bc7 100644
--- a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang
+++ b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang
@@ -6,7 +6,7 @@ import generated.activation_transform;
[[Variants("BiasT", "float, BFloat16")]]
[[Public]] [[Kernel]] func ActivationTransform<T, BiasT>(
[[Optional("!in_place")]] StructuredBuffer<T> input,
- [[Optional("!(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<T> fp_out,
+ [[Optional("(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)")]] RWStructuredBuffer<T> fp_out,
[[Optional("has_bias")]] StructuredBuffer<BiasT> bias,
[[Optional("(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int8_t> q_out,
[[Optional("(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<float> scales_out,
@@ -14,6 +14,7 @@ import generated.activation_transform;
StructuredBuffer<int> rht_factors,
uint batch_size,
uint element_count,
+ [[Optional("(ops == ActivationTransformOp::PackHalf)")]] RWStructuredBuffer<uint> packed_half_out,
[[Specialize]] ActivationTransformOp ops,
[[Specialize]] bool grouped_by_weight_nibble,
[[Specialize]] bool in_place,
diff --git a/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang b/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang
index 25889a958..3f12193ef 100644
--- a/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang
+++ b/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang
@@ -7,7 +7,7 @@ import generated.gated_act_mul;
[[Public]] [[Kernel]] func GatedActMul<T>(
StructuredBuffer<T> act_operand,
[[Optional("!interleaved")]] StructuredBuffer<T> value_operand,
- [[Optional("!(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<T> fp_out,
+ [[Optional("(ops == GatedActMulOp::FullPrecision)")]] RWStructuredBuffer<T> fp_out,
[[Optional("(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int8_t> q_out,
[[Optional("(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<float> scales_out,
[[Optional("(ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int> group_sums_out,
@@ -22,6 +22,7 @@ import generated.gated_act_mul;
[[Optional("clip_gate")]] float gate_clip_max,
[[Optional("clip_value")]] float value_clip_min,
[[Optional("clip_value")]] float value_clip_max,
+ [[Optional("(ops == GatedActMulOp::PackHalf)")]] RWStructuredBuffer<uint> packed_half_out,
[[Specialize]] GatedActMulOp ops,
[[Specialize]] bool grouped_by_weight_nibble,
[[Specialize]] bool interleaved,
diff --git a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
index c020b69c8..6ccb1a82f 100644
--- a/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/input_rht.rs
@@ -4,7 +4,10 @@ use super::{LinearInput, LinearInputPreparation};
use crate::{
backends::common::{
Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding,
- kernel::{ActivationTransform, matmul::ActivationFormat},
+ kernel::{
+ ActivationTransform,
+ matmul::{ActivationFormat, MatmulA, packed_half_bytes},
+ },
},
data_type::DataType,
};
@@ -24,11 +27,11 @@ impl<B: Backend> InputRht<B> {
) -> Result<Self, B::Error> {
let LinearInputPreparation {
rht_signs,
- activation_quantization,
+ activation_format,
} = preparation;
let rht = ActivationTransform::input_rht(context, data_type, in_place)?;
- let quantizer = activation_quantization
- .map(|quantization| ActivationTransform::quantize(context, data_type, quantization))
+ let quantizer = (activation_format != ActivationFormat::Bf16)
+ .then(|| ActivationTransform::quantize(context, data_type, activation_format))
.transpose()?;
Ok(Self {
@@ -45,14 +48,26 @@ impl<B: Backend> InputRht<B> {
format: ActivationFormat,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<LinearInput<B>, B::Error> {
- if format == ActivationFormat::Int8
- && let Some(quantizer) = &self.quantizer
- {
+ if let Some(quantizer) = self.quantizer.as_ref().filter(|quantizer| quantizer.format() == format) {
let input_dim = self.input_dim();
- let groups_per_row = input_dim.div_ceil(quantizer.scale_group_size());
+ if let ActivationFormat::PackedHalf {
+ group_size,
+ } = format
+ {
+ let mut values = command_buffer.allocate_scratch(packed_half_bytes(input_dim, group_size))?;
+ quantizer.encode_pack_half(input, &mut values, &self.rht_signs, batch_dim, input_dim, command_buffer);
+ return Ok(MatmulA::PackedHalf {
+ values,
+ group_size,
+ });
+ }
+ let ActivationFormat::Int8(quantization) = format else {
+ unreachable!()
+ };
+ let groups_per_row = input_dim.div_ceil(quantization.scale_group_size());
let mut values = command_buffer.allocate_scratch_for_shape(&[batch_dim, input_dim], DataType::I8)?;
let mut scales = command_buffer.allocate_scratch_for_shape(&[batch_dim, groups_per_row], DataType::F32)?;
- let mut group_sums = quantizer
+ let mut group_sums = quantization
.sum_group_size()
.map(|group_size| {
command_buffer
@@ -70,19 +85,18 @@ impl<B: Backend> InputRht<B> {
command_buffer,
);
- return Ok(LinearInput::Int8Symmetric {
+ return Ok(MatmulA::Int8Symmetric {
values,
scales,
group_sums,
- scale_group_size: quantizer.scale_group_size(),
- code_layout: quantizer.code_layout(),
+ quantization,
});
}
let input_dim = self.input_dim();
let mut transformed = command_buffer.allocate_scratch(input.size())?;
self.rht.encode_fp(input, &mut transformed, &self.rht_signs, batch_dim, input_dim, command_buffer);
- Ok(LinearInput::FullPrecision(transformed))
+ Ok(MatmulA::full_precision(transformed))
}
pub(super) fn prepare_in_place(
@@ -92,13 +106,13 @@ impl<B: Backend> InputRht<B> {
format: ActivationFormat,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<LinearInput<B>, B::Error> {
- if format == ActivationFormat::Int8 && self.quantizer.is_some() {
+ if self.quantizer.as_ref().is_some_and(|kernel| kernel.format() == format) {
return self.prepare(&input, batch_dim, format, command_buffer);
}
let input_dim = self.input_dim();
self.rht.encode_fp_in_place(&mut input, &self.rht_signs, None, batch_dim, input_dim, command_buffer);
- Ok(LinearInput::FullPrecision(input))
+ Ok(MatmulA::full_precision(input))
}
fn input_dim(&self) -> u32 {
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index b86a60475..9e2679be5 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -6,7 +6,7 @@ use crate::{
backends::common::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding,
kernel::{
- ActivationQuantization, Kernels,
+ Kernels,
matmul::{
ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,
MatmulShape,
@@ -46,6 +46,7 @@ pub struct LinearMatmul<B: Backend> {
input_dim: u32,
output_dim: u32,
output_data_type: DataType,
+ prepared_activation_format: ActivationFormat,
}
async fn load_biases<B: Backend>(
@@ -110,17 +111,23 @@ impl<B: Backend> LinearMatmul<B> {
input_dim,
output_dim,
output_data_type,
+ prepared_activation_format: ActivationFormat::Bf16,
})
}
- pub(super) fn optimize_weights_layout(
+ pub(super) fn prepare_activation(
&mut self,
context: &B::Context,
- ) -> Result<Option<ActivationQuantization>, B::Error> {
- let shape = self.matmul_shape(1, false);
- let kernel = self.kernel.lock();
- self.matrix.optimize_weights_layout(|weights| kernel.optimize_weights_layout(&shape, weights, context))?;
- Ok(kernel.select_activation_quantization(&self.matmul_shape(1, false), context))
+ ) -> Result<ActivationFormat, B::Error> {
+ {
+ let shape = self.matmul_shape(1, ActivationFormat::Bf16);
+ let kernel = self.kernel.lock();
+ self.matrix.optimize_weights_layout(|weights| kernel.optimize_weights_layout(&shape, weights, context))?;
+ }
+ let shape = self.matmul_shape(1, ActivationFormat::Bf16);
+ self.prepared_activation_format =
+ self.kernel.lock().activation_preparation(&shape, self.matmul_b().weights_layout(), context);
+ Ok(self.prepared_activation_format)
}
pub(super) fn encode_with_a(
@@ -159,7 +166,7 @@ impl<B: Backend> LinearMatmul<B> {
fn matmul_shape(
&self,
batch_dim: u32,
- a_full_precision: bool,
+ a: ActivationFormat,
) -> MatmulShape {
let b = self.matmul_b();
MatmulShape {
@@ -173,7 +180,7 @@ impl<B: Backend> LinearMatmul<B> {
b_bits: b.bits_per_b(),
b_group_size: b.group_size(),
signed_codes: b.signed_codes(),
- a_full_precision,
+ a,
gathered: false,
params_layout: b.quant_params_layout(),
d_transform: self.d_ops().mask(),
@@ -200,7 +207,7 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
- self.encode_with_a(input.as_matmul_a(), batch_dim, None::<Gather<&B::ScratchBuffer>>, command_buffer)
+ self.encode_with_a(input.as_ref(), batch_dim, None::<Gather<&B::ScratchBuffer>>, command_buffer)
}
fn select_activation_format(
@@ -208,7 +215,12 @@ impl<B: Backend> Linear<B> for LinearMatmul<B> {
batch_dim: u32,
context: &B::Context,
) -> ActivationFormat {
- let bf16_shape = self.matmul_shape(batch_dim, true);
- self.kernel.lock().select_activation_format(&bf16_shape, self.matmul_b().weights_layout(), context)
+ let shape = self.matmul_shape(batch_dim, ActivationFormat::Bf16);
+ self.kernel.lock().select_activation_format(
+ &shape,
+ self.matmul_b().weights_layout(),
+ self.prepared_activation_format,
+ context,
+ )
}
}
diff --git a/crates/uzu-engine/src/encodable_block/linear/mod.rs b/crates/uzu-engine/src/encodable_block/linear/mod.rs
index 73f52ef53..ad96f26b5 100644
--- a/crates/uzu-engine/src/encodable_block/linear/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/mod.rs
@@ -14,10 +14,7 @@ use crate::{
backends::common::{
Backend, BufferRef, CommandBuffer, Kernels,
gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
- kernel::{
- ActivationQuantization,
- matmul::{ActivationFormat, Int8CodeLayout, MatmulA, MatmulKernel},
- },
+ kernel::matmul::{ActivationFormat, MatmulA, MatmulKernel},
},
config::weight_matrix::{
AnyWeightMatrixSpec,
@@ -42,7 +39,7 @@ pub trait Linear<B: Backend>: Send + Sync {
batch_dim: u32,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
- self.encode_input(LinearInput::FullPrecision(input), batch_dim, command_buffer)
+ self.encode_input(MatmulA::full_precision(input), batch_dim, command_buffer)
}
fn encode_input(
@@ -61,49 +58,16 @@ pub trait Linear<B: Backend>: Send + Sync {
}
}
-pub enum LinearInput<B: Backend> {
- FullPrecision(B::ScratchBuffer),
- Int8Symmetric {
- values: B::ScratchBuffer,
- scales: B::ScratchBuffer,
- group_sums: Option<B::ScratchBuffer>,
- scale_group_size: u32,
- code_layout: Int8CodeLayout,
- },
-}
+pub type LinearInput<B> = MatmulA<<B as Backend>::ScratchBuffer>;
pub struct Gather<T: BufferRef> {
pub indices: T,
pub output_dim: u32,
}
-impl<B: Backend> LinearInput<B> {
- fn as_matmul_a(&self) -> MatmulA<&B::ScratchBuffer> {
- match self {
- Self::FullPrecision(values) => MatmulA::FullPrecision {
- values,
- offset: 0,
- },
- Self::Int8Symmetric {
- values,
- scales,
- group_sums,
- scale_group_size,
- code_layout,
- } => MatmulA::Int8Symmetric {
- values,
- scales,
- group_sums: group_sums.as_ref(),
- scale_group_size: *scale_group_size,
- code_layout: *code_layout,
- },
- }
- }
-}
-
pub struct LinearInputPreparation<B: Backend> {
pub rht_signs: B::GlobalGpuBuffer,
- pub activation_quantization: Option<ActivationQuantization>,
+ pub activation_format: ActivationFormat,
}
#[derive(Debug, Error)]
diff --git a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
index f26308b16..6fdba1bde 100644
--- a/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/rht_wrapper.rs
@@ -3,7 +3,9 @@ use thiserror::Error;
use crate::{
backends::common::{
- Backend, CommandBuffer, CommandBufferEncoding, gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ Backend, CommandBuffer, CommandBufferEncoding,
+ gpu_types::hadamard_order::HADAMARD_TRANSFORM_BLOCK_SIZE,
+ kernel::{ActivationFormat, matmul::MatmulA},
},
config::weight_matrix::{
AnyWeightMatrixSpec,
@@ -62,7 +64,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
return Err(RHTLinearWrapperError::UnsupportedConfiguration(format!("{spec:?}")));
}
- let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
+ let (rht_signs, inner_linear, activation_format) = Self::load_inner_with_output_rht(
input_dimension,
output_dimension,
weights_data_type,
@@ -73,13 +75,11 @@ impl<B: Backend> RHTLinearWrapper<B> {
context,
)
.await?;
- let activation_quantization =
- inner_linear.optimize_weights_layout(context).map_err(RHTLinearWrapperError::BackendError)?;
Self::build_self_contained(
input_data_type,
LinearInputPreparation {
rht_signs,
- activation_quantization,
+ activation_format,
},
inner_linear,
context,
@@ -103,7 +103,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
return Ok(None);
}
- let (rht_signs, mut inner_linear) = Self::load_inner_with_output_rht(
+ let (rht_signs, inner_linear, activation_format) = Self::load_inner_with_output_rht(
input_dimension,
output_dimension,
weights_data_type,
@@ -116,11 +116,9 @@ impl<B: Backend> RHTLinearWrapper<B> {
.await?;
let input_preparation = LinearInputPreparation {
rht_signs,
- activation_quantization: inner_linear
- .optimize_weights_layout(context)
- .map_err(RHTLinearWrapperError::BackendError)?,
+ activation_format,
};
- if input_preparation.activation_quantization.is_some() && !allow_prequantized_activation {
+ if input_preparation.activation_format != ActivationFormat::Bf16 && !allow_prequantized_activation {
let wrapper = Self::build_self_contained(input_data_type, input_preparation, inner_linear, context)?;
Ok(Some((Box::new(wrapper), None)))
} else {
@@ -137,7 +135,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
has_biases: bool,
parameter_tree: &ParameterTree<'_, B>,
context: &B::Context,
- ) -> Result<(B::GlobalGpuBuffer, LinearMatmul<B>), RHTLinearWrapperError<B>> {
+ ) -> Result<(B::GlobalGpuBuffer, LinearMatmul<B>, ActivationFormat), RHTLinearWrapperError<B>> {
let weights_tree = parameter_tree.subtree("weights");
let quantized_weights_tree = weights_tree.subtree("quantized");
let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
@@ -145,7 +143,7 @@ impl<B: Backend> RHTLinearWrapper<B> {
weights_tree.read_buffer("incoherence_signs.input_signs", &[input_dimension], DataType::I32).await?;
let output_factors =
weights_tree.read_buffer("incoherence_signs.output_signs", &[output_dimension], DataType::I32).await?;
- let inner_linear = LinearMatmul::load(
+ let mut inner_linear = LinearMatmul::load(
input_dimension,
output_dimension,
weights_data_type,
@@ -158,7 +156,9 @@ impl<B: Backend> RHTLinearWrapper<B> {
context,
)
.await?;
- Ok((rht_signs, inner_linear))
+ let activation_format =
+ inner_linear.prepare_activation(context).map_err(RHTLinearWrapperError::BackendError)?;
+ Ok((rht_signs, inner_linear, activation_format))
}
fn build_self_contained(
@@ -185,7 +185,10 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
let input = match input {
- LinearInput::FullPrecision(input) => input,
+ MatmulA::FullPrecision {
+ values: input,
+ ..
+ } => input,
input => return self.inner_linear.encode_input(input, batch_dim, command_buffer),
};
let command_buffer = &mut *command_buffer.debug_group("linear (rht)");
@@ -196,7 +199,7 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
.prepare_in_place(input, batch_dim, format, command_buffer)
.map_err(LinearEncodeError::Backend)?;
let output = self.inner_linear.encode_with_a(
- input.as_matmul_a(),
+ input.as_ref(),
batch_dim,
None::<Gather<&B::ScratchBuffer>>,
command_buffer,
diff --git a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
index 33bf030f8..e474d2c10 100644
--- a/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/untied_readout.rs
@@ -60,11 +60,10 @@ impl<B: Backend> UntiedReadout<B> {
.await?;
let rht_signs =
tree.subtree("incoherence_signs").read_buffer("input_signs", &[model_dim], DataType::I32).await?;
+ let activation_format = linear.prepare_activation(context).map_err(LinearMatmulError::BackendError)?;
let preparation = LinearInputPreparation {
rht_signs,
- activation_quantization: linear
- .optimize_weights_layout(context)
- .map_err(LinearMatmulError::BackendError)?,
+ activation_format,
};
let input_rht = InputRht::new(data_type, preparation, /* in_place */ false, context)
.map_err(LinearMatmulError::BackendError)?;
@@ -83,10 +82,7 @@ impl<B: Backend> UntiedReadout<B> {
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, LinearEncodeError<B>> {
let Some(input_rht) = &self.input_rht else {
- let a = MatmulA::FullPrecision {
- values: input,
- offset: 0,
- };
+ let a = MatmulA::full_precision(input);
return self.linear.encode_with_a(a, batch_dim, gather, command_buffer);
};
@@ -96,6 +92,6 @@ impl<B: Backend> UntiedReadout<B> {
self.linear.select_activation_format(batch_dim, command_buffer.context())
};
let input = input_rht.prepare(input, batch_dim, format, command_buffer).map_err(LinearEncodeError::Backend)?;
- self.linear.encode_with_a(input.as_matmul_a(), batch_dim, gather, command_buffer)
+ self.linear.encode_with_a(input.as_ref(), batch_dim, gather, command_buffer)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
index a97d1abaa..2a7847bc0 100644
--- a/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
+++ b/crates/uzu-engine/src/encodable_block/mlp/gate_act_mul.rs
@@ -2,7 +2,10 @@ use crate::{
backends::common::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding,
gpu_types::activation_type::ActivationType,
- kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::ActivationFormat},
+ kernel::{
+ GatedActMul, GatedActMulSettings,
+ matmul::{ActivationFormat, MatmulA, packed_half_bytes},
+ },
},
config::{activation::AnyActivation, clipping::ClippingBounds},
data_type::DataType,
@@ -15,7 +18,6 @@ pub struct MlpGateActMulEncodable<B: Backend> {
hidden_dim: u32,
data_type: DataType,
hadamard_factors: Option<B::GlobalGpuBuffer>,
- activation_quantization: Option<ActivationQuantization>,
quantized_kernel: Option<GatedActMul<B>>,
}
@@ -29,16 +31,17 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
input_preparation: Option<LinearInputPreparation<B>>,
context: &B::Context,
) -> Result<Self, B::Error> {
- let (hadamard_factors, activation_quantization) = input_preparation
- .map_or((None, None), |preparation| (Some(preparation.rht_signs), preparation.activation_quantization));
+ let (hadamard_factors, format) = input_preparation.map_or((None, ActivationFormat::Bf16), |preparation| {
+ (Some(preparation.rht_signs), preparation.activation_format)
+ });
let settings = GatedActMulSettings {
activation_alpha: activation.custom_alpha(),
gate_clipping,
value_clipping,
};
let fp_kernel = GatedActMul::full_precision(context, data_type, true, hadamard_factors.is_some(), settings)?;
- let quantized_kernel = activation_quantization
- .map(|quantization| GatedActMul::quantized(context, data_type, quantization, settings))
+ let quantized_kernel = (format != ActivationFormat::Bf16)
+ .then(|| GatedActMul::quantized(context, data_type, format, settings))
.transpose()?;
Ok(Self {
fp_kernel,
@@ -46,7 +49,6 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
hidden_dim,
data_type,
hadamard_factors,
- activation_quantization,
quantized_kernel,
})
}
@@ -61,10 +63,31 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
}
- let input = if act_format == ActivationFormat::Int8
- && let Some(quantization) = self.activation_quantization
- {
- let kernel = self.quantized_kernel.as_ref().expect("INT8 input requires a quantized gate kernel");
+
+ if let Some(kernel) = self.quantized_kernel.as_ref().filter(|kernel| kernel.format() == act_format) {
+ let hadamard_factors = self.hadamard_factors.as_ref().expect("quantized input requires RHT factors");
+ if let ActivationFormat::PackedHalf {
+ group_size,
+ } = act_format
+ {
+ let mut values = command_buffer.allocate_scratch(packed_half_bytes(self.hidden_dim, group_size))?;
+ kernel.encode_pack_half(
+ fused_up,
+ &mut values,
+ hadamard_factors,
+ self.hidden_dim,
+ batch_dim,
+ self.activation.act_type(),
+ command_buffer,
+ );
+ return Ok(MatmulA::PackedHalf {
+ values,
+ group_size,
+ });
+ }
+ let ActivationFormat::Int8(quantization) = act_format else {
+ unreachable!()
+ };
let mut values = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], DataType::I8)?;
let mut scales = command_buffer.allocate_scratch_for_shape(
&[batch_dim, self.hidden_dim.div_ceil(quantization.scale_group_size())],
@@ -82,37 +105,33 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
&mut values,
&mut scales,
group_sums.as_mut(),
- self.hadamard_factors.as_ref().expect("INT8 input requires RHT factors"),
+ hadamard_factors,
self.hidden_dim,
batch_dim,
self.activation.act_type(),
command_buffer,
);
- LinearInput::Int8Symmetric {
+ return Ok(MatmulA::Int8Symmetric {
values,
scales,
group_sums,
- scale_group_size: quantization.scale_group_size(),
- code_layout: quantization.code_layout(),
- }
- } else {
- let mut hidden =
- command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
- self.fp_kernel.encode_fp(
- fused_up,
- None::<&B::ScratchBuffer>,
- &mut hidden,
- self.hadamard_factors.as_ref(),
- self.hidden_dim,
- batch_dim,
- 0,
- 0,
- self.activation.act_type(),
- command_buffer,
- );
- LinearInput::FullPrecision(hidden)
- };
+ quantization,
+ });
+ }
- Ok(input)
+ let mut hidden = command_buffer.allocate_scratch_for_shape(&[batch_dim, self.hidden_dim], self.data_type)?;
+ self.fp_kernel.encode_fp(
+ fused_up,
+ None::<&B::ScratchBuffer>,
+ &mut hidden,
+ self.hadamard_factors.as_ref(),
+ self.hidden_dim,
+ batch_dim,
+ 0,
+ 0,
+ self.activation.act_type(),
+ command_buffer,
+ );
+ Ok(MatmulA::full_precision(hidden))
}
}
diff --git a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
index 4196fee4d..e6d5c858f 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/activation_transform_test.rs
@@ -157,7 +157,7 @@ mod quantize {
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- kernel::{ActivationQuantization, ActivationTransform, matmul::Int8CodeLayout},
+ kernel::{ActivationFormat, ActivationQuantization, ActivationTransform, matmul::Int8CodeLayout},
},
cpu::Cpu,
},
@@ -187,13 +187,15 @@ mod quantize {
let kernel = ActivationTransform::<B>::quantize(
context.as_ref(),
DataType::F32,
- ActivationQuantization::new(
- scale_group_size,
- sum_group_size.unwrap_or(scale_group_size),
- emit_group_sums,
- code_layout,
- )
- .expect("supported activation quantization"),
+ ActivationFormat::Int8(
+ ActivationQuantization::new(
+ scale_group_size,
+ sum_group_size.unwrap_or(scale_group_size),
+ emit_group_sums,
+ code_layout,
+ )
+ .expect("supported activation quantization"),
+ ),
)
.expect("quantize transform");
let mut command_buffer = context.as_ref().create_command_buffer(None, None).expect("command buffer");
diff --git a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
index cceaec422..c7adf46f4 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/gated_act_mul_test.rs
@@ -10,7 +10,9 @@ use crate::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
gpu_types::activation_type::ActivationType,
- kernel::{ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::Int8CodeLayout},
+ kernel::{
+ ActivationFormat, ActivationQuantization, GatedActMul, GatedActMulSettings, matmul::Int8CodeLayout,
+ },
},
cpu::Cpu,
},
@@ -125,8 +127,10 @@ fn run_nibble_grouped_quantized<B: Backend>(input: &InterleavedInput<bf16>) -> (
let kernel = GatedActMul::<B>::quantized(
&context,
DataType::BF16,
- ActivationQuantization::new(128, 128, false, Int8CodeLayout::GroupedByNibble)
- .expect("supported activation quantization"),
+ ActivationFormat::Int8(
+ ActivationQuantization::new(128, 128, false, Int8CodeLayout::GroupedByNibble)
+ .expect("supported activation quantization"),
+ ),
GatedActMulSettings::default(),
)
.expect("create quantized GatedActMul");
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
index 236352bf0..86a3ed3fa 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/a8w_bench.rs
@@ -13,7 +13,7 @@ use crate::{
quantization_method::QuantizationMethod,
},
kernel::{
- ActivationQuantization, ActivationTransform, Kernels,
+ ActivationFormat, ActivationQuantization, ActivationTransform, Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
matmul::{
Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,
@@ -135,10 +135,7 @@ impl BenchmarkData {
&'a <Metal as Backend>::GlobalGpuBuffer,
> {
MatmulArguments {
- a: MatmulA::FullPrecision {
- values: &self.a_working,
- offset: 0,
- },
+ a: MatmulA::full_precision(&self.a_working),
b: MatmulB::Quantized(QuantizedB {
codes: &self.unsigned_weights,
scales: &self.weight_scales,
@@ -171,6 +168,9 @@ fn encode_step(
) {
match path {
BenchPath::A8GemmMxu => {
+ let ActivationFormat::Int8(quantization) = prepare.format() else {
+ unreachable!("A8 benchmark requires Int8 activation preparation")
+ };
prepare.encode_quantize(
&data.activations,
&mut data.a_int8,
@@ -192,9 +192,7 @@ fn encode_step(
values: &data.a_int8,
scales: &data.a_scales,
group_sums: None,
- scale_group_size: 128,
- code_layout: Int8CodeLayout::for_right_bits(DataType::from(data.mode).size_in_bits() as u32)
- .expect("W4/W8 benchmark"),
+ quantization,
},
b: MatmulB::Quantized(QuantizedB {
codes: &data.a8_weights,
@@ -287,8 +285,10 @@ fn bench_a8w(c: &mut Criterion) {
let prepare = ActivationTransform::<Metal>::quantize(
&context,
DataType::BF16,
- ActivationQuantization::new(128, 128, false, Int8CodeLayout::for_right_bits(bits).expect("W8 benchmark"))
- .expect("supported activation quantization"),
+ ActivationFormat::Int8(
+ ActivationQuantization::new(128, 128, false, Int8CodeLayout::for_right_bits(bits).expect("W8 benchmark"))
+ .expect("supported activation quantization"),
+ ),
)
.expect("prepare kernel");
bench_bits(c, &context, &prepare, &hadamard, bits);
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index e53936048..913832e7e 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -3,38 +3,18 @@ use metal::MTLGPUFamily;
use uzu_engine_macros::uzu_test;
use super::*;
-use crate::backends::{
- common::{
- Context,
- gpu_types::gemm::GemmDTransform,
- kernel::matmul::{MatmulKernel, QuantParamsLayout},
+use crate::{
+ backends::{
+ common::{
+ Context,
+ gpu_types::gemm::GemmDTransform,
+ kernel::matmul::{MatmulKernel, QuantParamsLayout},
+ },
+ metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},
},
- metal::{context::MetalContext, kernel::matmul::MatmulMetalKernel},
+ tests::matmul::{a8_format, matmul_shape as shape, quantized_matmul_shape},
};
-fn shape(
- m: u32,
- n: u32,
- k: u32,
-) -> MatmulShape {
- MatmulShape {
- m,
- n,
- k,
- b_transpose: true,
- b_leading_dimension: None,
- b_prologue: GemmBPrologueKind::FullPrecision,
- b_is_trellis: false,
- b_bits: None,
- b_group_size: None,
- signed_codes: false,
- a_full_precision: true,
- gathered: false,
- params_layout: Some(QuantParamsLayout::OutputGroup),
- d_transform: GemmDTransform::empty(),
- }
-}
-
fn quant(mut shape: MatmulShape) -> MatmulShape {
shape.b_prologue = GemmBPrologueKind::ScaleSymmetricDequant;
shape.b_bits = Some(4);
@@ -118,7 +98,7 @@ fn selection_fallbacks_and_split_k_are_preserved() {
let mut invalid_layout = quant(shape(64, 4096, 4096));
invalid_layout.b_transpose = false;
assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Simdgroup);
- invalid_layout.a_full_precision = false;
+ invalid_layout.a = a8_format();
assert_eq!(problem(invalid_layout, DataType::BF16).select_plan().engine, Mxu);
assert_eq!(problem(quant(shape(64, 4096, 4095)), DataType::BF16).select_plan().engine, Simdgroup);
@@ -129,14 +109,14 @@ fn selection_fallbacks_and_split_k_are_preserved() {
(8, 4096, 384, Tile16x32x256_Simdgroups1x1, 1),
] {
let mut a8 = quant(shape(m, n, k));
- a8.a_full_precision = false;
+ a8.a = a8_format();
let plan = problem(a8, DataType::BF16).select_plan();
assert_eq!(plan.tiling, expected_tiling);
assert_eq!(plan.split_k, expected_split_k);
}
let mut biased = quant(shape(16, 4096, 4096));
- biased.a_full_precision = false;
+ biased.a = a8_format();
biased.d_transform = GemmDTransform::BIAS;
assert_eq!(
GemmProblem::new(biased, DataType::BF16, DataType::F32, true, MTLGPUFamily::Apple7).select_plan().split_k,
@@ -166,7 +146,7 @@ fn trellis_plan_matches_projection_cases() {
(1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),
] {
let mut trellis_shape = shape(m, n, k);
- trellis_shape.a_full_precision = false;
+ trellis_shape.a = ActivationFormat::Trellis;
trellis_shape.b_is_trellis = true;
let plan = problem(trellis_shape, DataType::BF16).select_plan();
assert_eq!((plan.engine, plan.tiling, plan.split_k), (GemmEngine::Mxu, tiling, split_k), "M {m} N {n} K {k}");
@@ -224,22 +204,7 @@ fn int8_activations_require_mxu_tiled_weights() {
(64, 20480, 8, true),
(64, 20608, 8, false),
] {
- let shape = MatmulShape {
- b_prologue: if bits == 4 {
- GemmBPrologueKind::ScaleZeroPointDequant
- } else {
- GemmBPrologueKind::ScaleSymmetricDequant
- },
- b_bits: Some(bits),
- signed_codes: bits == 8,
- a_full_precision: false,
- params_layout: Some(QuantParamsLayout::GroupOutput),
- ..quant(shape(16, n, k))
- };
- assert_eq!(
- kernel.select_activation_quantization(&shape, &context).is_some(),
- expected,
- "n={n} k={k} bits={bits}"
- );
+ let shape = quantized_matmul_shape(16, n, k, bits, 64);
+ assert_eq!(kernel.select_activation_quantization(&shape, &context).is_some(), expected, "n={n} k={k} W{bits}");
}
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
index 6072d4322..da07743a8 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
@@ -1,19 +1,15 @@
-use metal::MTLGPUFamily;
use uzu_engine_macros::uzu_test;
-use super::*;
+use super::{DEFAULT_RESULTS_PER_SIMDGROUP, GemvTile, fp_tile, quantized_tile, tile};
use crate::{
backends::{
- common::{
- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
- kernel::matmul::{MatmulShape, QuantParamsLayout},
- },
+ common::kernel::matmul::{MatmulShape, QuantParamsLayout},
metal::gpu_types::gemm::GemmBLayout,
},
data_type::DataType,
+ tests::matmul::quantized_matmul_shape as shape,
};
-const SMALL_GPU_CORE_COUNT: u32 = 29;
const LARGE_GPU_CORE_COUNT: u32 = 30;
fn qtile(
@@ -26,121 +22,52 @@ fn qtile(
#[uzu_test]
fn fp_policy_cases() {
let cases = [
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 12288, 1536, true, tile(8, 8, 1)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 12288, 1536, false, tile(8, 1, 1)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, true, tile(8, 1, 1)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, true, tile(8, 2, 1)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 8, 12288, 1536, true, tile(8, 1, 4)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 12288, true, tile(8, 8, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 262144, 1536, true, tile(8, 1, 4)),
+ (1, 12288, 1536, true, tile(8, 8, 1)),
+ (1, 12288, 1536, false, tile(8, 1, 1)),
+ (1, 1536, 256, true, tile(8, 2, 1)),
+ (8, 12288, 1536, true, tile(8, 1, 4)),
+ (1, 1536, 12288, true, tile(8, 8, 1)),
+ (1, 262144, 1536, true, tile(8, 8, 1)),
];
- for (gpu_core_count, apple_gpu_family, m, n, k, aligned, expected) in cases {
- assert_eq!(fp_tile(gpu_core_count, apple_gpu_family, m, n, k, aligned), expected);
+ for (m, n, k, aligned, expected) in cases {
+ assert_eq!(fp_tile(m, n, k, aligned), expected, "m={m} n={n} k={k} aligned={aligned}");
}
}
#[uzu_test]
fn quant_policy_cases() {
let cases = [
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 256, 1536, 4, qtile(2, 1)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 262144, 1536, 4, qtile(8, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 1536, 256, 4, qtile(4, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 1, 1536, 256, 4, qtile(8, 4)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple8, 1, 2048, 1536, 4, qtile(8, 2)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 256, 1536, 4, qtile(8, 2)),
- (SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple7, 1, 1536, 256, 4, qtile(4, 8)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 2, 2048, 1536, 4, qtile(8, 4)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 5120, 5120, 4, qtile(8, 4)),
- (LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 1, 2048, 1536, 8, GemvTile::quantized(8, 4, 1, 32, 4)),
+ (1, 256, 4, qtile(8, 4)),
+ (1, 262144, 4, qtile(8, 4)),
+ (2, 2048, 4, qtile(8, 4)),
+ (4, 5120, 4, qtile(8, 4)),
+ (1, 2048, 8, GemvTile::quantized(8, 4, 1, 32, 4)),
];
- for (gpu_core_count, apple_gpu_family, m, n, k, bits, expected) in cases {
- let actual =
- quantized_tile(gpu_core_count, apple_gpu_family, bits, 32, m, n, k, true).expect("quantized route");
- assert_eq!(
- actual, expected,
- "m={m} n={n} k={k} bits={bits} gpu_core_count={gpu_core_count} apple_gpu_family={apple_gpu_family:?}"
- );
+ for (m, n, bits, expected) in cases {
+ let actual = quantized_tile(bits, 32, m, n, true).expect("quantized route");
+ assert_eq!(actual, expected, "m={m} n={n} bits={bits}");
}
}
-#[uzu_test]
-fn gpu_core_count_boundary_selects_large_policy() {
- let small = quantized(SMALL_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("small GPU route");
- let large = quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple9, 4, 32, 1, 256, 1536).expect("large GPU route");
-
- assert_eq!(small, qtile(4, 2));
- assert_eq!(large, qtile(2, 1));
-}
-
-fn quantized(
- gpu_core_count: u32,
- apple_gpu_family: MTLGPUFamily,
- bits: u32,
- group: u32,
- m: u32,
- n: u32,
- k: u32,
-) -> Option<GemvTile> {
- quantized_tile(gpu_core_count, apple_gpu_family, bits, group, m, n, k, true)
-}
-
#[uzu_test]
fn quantized_policy_edges() {
- assert_eq!(
- quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 64, 4, 8192, 4100),
- Some(GemvTile::quantized(8, 4, 1, 32, 4))
- );
- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8, 4096).is_some());
- assert!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 1, 8192, 4096).is_some());
- assert_eq!(quantized(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 9, 8192, 4096), None);
- assert_eq!(
- quantized_tile(LARGE_GPU_CORE_COUNT, MTLGPUFamily::Apple10, 4, 32, 4, 8192, 4096, true),
- Some(GemvTile::quantized(8, 4, 1, 32, 2))
- );
-}
-
-fn quant_shape(
- m: u32,
- n: u32,
- d_transform: GemmDTransform,
-) -> MatmulShape {
- MatmulShape {
- m,
- n,
- k: 4096,
- b_transpose: true,
- b_leading_dimension: None,
- b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,
- b_is_trellis: false,
- b_bits: Some(4),
- b_group_size: Some(32),
- signed_codes: false,
- a_full_precision: true,
- gathered: false,
- params_layout: Some(QuantParamsLayout::OutputGroup),
- d_transform,
- }
+ assert_eq!(quantized_tile(4, 64, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 4)));
+ assert!(quantized_tile(4, 32, 4, 8, true).is_some());
+ assert!(quantized_tile(4, 32, 1, 8192, true).is_some());
+ assert_eq!(quantized_tile(4, 32, 9, 8192, true), None);
+ assert_eq!(quantized_tile(4, 32, 6, 8192, true), None);
+ assert_eq!(quantized_tile(4, 32, 6, 64, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));
+ assert_eq!(quantized_tile(4, 32, 1, 8192, false), None);
+ assert_eq!(quantized_tile(4, 32, 4, 8192, true), Some(GemvTile::quantized(8, 4, 1, 32, 2)));
}
#[uzu_test]
fn block_unaligned_quantized_k_stays_on_gemv() {
let shape = MatmulShape {
- m: 1,
- n: 64,
- k: 1152,
- b_transpose: true,
- b_leading_dimension: None,
- b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,
- b_is_trellis: false,
- b_bits: Some(4),
- b_group_size: Some(64),
- signed_codes: false,
- a_full_precision: true,
- gathered: false,
params_layout: Some(QuantParamsLayout::OutputGroup),
- d_transform: GemmDTransform::empty(),
+ ..shape(1, 64, 1152, 4, 64)
};
assert!(
super::super::kernel::GemvSpecialization::select_shape(
@@ -150,7 +77,6 @@ fn block_unaligned_quantized_k_stays_on_gemv() {
DataType::BF16,
DataType::BF16,
LARGE_GPU_CORE_COUNT,
- MTLGPUFamily::Apple10,
)
.is_some()
);
@@ -158,36 +84,29 @@ fn block_unaligned_quantized_k_stays_on_gemv() {
#[uzu_test]
fn specialization_preserves_quantized_route() {
- let select = |n, transform| {
+ let select = |n| {
super::super::kernel::GemvSpecialization::select_shape(
- &quant_shape(4, n, transform),
+ &MatmulShape {
+ params_layout: Some(QuantParamsLayout::OutputGroup),
+ ..shape(4, n, 4096, 4, 32)
+ },
GemmBLayout::RowMajor,
DataType::BF16,
DataType::BF16,
DataType::BF16,
LARGE_GPU_CORE_COUNT,
- MTLGPUFamily::Apple10,
)
};
- let clean = select(8192, GemmDTransform::empty()).expect("quantized specialization");
+ let clean = select(8192).expect("quantized specialization");
assert!(clean.output_row_tile() > DEFAULT_RESULTS_PER_SIMDGROUP);
- assert!(select(8192 + clean.output_row_tile() / 2, GemmDTransform::empty()).is_some());
+ assert!(select(8192 + clean.output_row_tile() / 2).is_some());
}
#[uzu_test]
fn gathered_group_major_is_a_gemv_route() {
- let mut shape = quant_shape(1, 8192, GemmDTransform::empty());
- shape.params_layout = Some(QuantParamsLayout::GroupOutput);
+ let mut shape = shape(1, 8192, 4096, 4, 32);
shape.gathered = true;
assert!(
- super::super::kernel::GemvSpecialization::select_tile(
- &shape,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- qtile(8, 4),
- )
- .is_some()
+ super::super::kernel::GemvSpecialization::select_tile(&shape, GemmBLayout::RowMajor, qtile(8, 4),).is_some()
);
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
deleted file mode 100644
index c9e647499..000000000
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
+++ /dev/null
@@ -1,291 +0,0 @@
-use metal::MTLGPUFamily;
-use uzu_engine_macros::uzu_test;
-use xxhash_rust::xxh3::xxh3_64;
-
-use super::*;
-use crate::{
- backends::{
- common::{
- gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
- kernel::matmul::{MatmulShape, QuantParamsLayout},
- },
- metal::{
- gpu_types::gemm::GemmBLayout,
- kernel::matmul::{MatmulDispatch, MatmulMetalKernel, gemv::GemvSpecialization},
- },
- },
- data_type::DataType,
-};
-
-const FROZEN_PLAN_FINGERPRINT: u64 = 10_794_242_881_707_148_570;
-
-const DEVICES: [(&str, &str, u32, MTLGPUFamily, bool); 7] = [
- ("m1", "Apple M1", 8, MTLGPUFamily::Apple7, false),
- ("m2", "Apple M2", 10, MTLGPUFamily::Apple8, false),
- ("m2-pro", "Apple M2 Pro", 19, MTLGPUFamily::Apple8, false),
- ("m3-max", "Apple M3 Max", 40, MTLGPUFamily::Apple9, false),
- ("m4", "Apple M4", 10, MTLGPUFamily::Apple9, false),
- ("m4-pro", "Apple M4 Pro", 20, MTLGPUFamily::Apple9, false),
- ("m5-max", "Apple M5 Max", 40, MTLGPUFamily::Apple10, true),
-];
-const FORMATS: [(&str, u32, u32, GemmBPrologueKind); 2] = [
- ("W4/ZP G64", 4, 64, GemmBPrologueKind::ScaleZeroPointDequant),
- ("W8/Symmetric G64", 8, 64, GemmBPrologueKind::ScaleSymmetricDequant),
-];
-const SHAPES: [(&str, u32, u32); 6] = [
- ("down", 5120, 17408),
- ("gate", 6144, 5120),
- ("gate-up", 34816, 5120),
- ("projection-in", 16480, 5120),
- ("projection-out", 5120, 6144),
- ("readout", 248320, 5120),
-];
-
-fn problem(
- m: u32,
- n: u32,
- k: u32,
- bits: u32,
- group: u32,
- prologue: GemmBPrologueKind,
-) -> MatmulShape {
- MatmulShape {
- m,
- n,
- k,
- b_transpose: true,
- b_leading_dimension: None,
- b_prologue: prologue,
- b_is_trellis: false,
- b_bits: Some(bits),
- b_group_size: Some(group),
- signed_codes: false,
- a_full_precision: true,
- gathered: false,
- params_layout: Some(QuantParamsLayout::OutputGroup),
- d_transform: GemmDTransform::empty(),
- }
-}
-
-#[uzu_test]
-fn table_is_complete_and_fingerprint_is_stable() {
- let mut canonical = Vec::new();
- let mut matched_rows = vec![false; ROWS.len()];
- for &(device_label, device_name, gpu_core_count, apple_gpu_family, supports_mxu) in &DEVICES {
- for &(format_name, bits, group, prologue) in &FORMATS {
- for m in 2..=7 {
- for &(shape_name, n, k) in SHAPES.iter().filter(|shape| bits == 4 || shape.0 != "gate") {
- let mask = shape(n, k);
- let matches: Vec<_> = ROWS
- .iter()
- .enumerate()
- .filter(|row| {
- row.1.device_name == device_name
- && row.1.apple_gpu_family == apple_gpu_family
- && row.1.bits == bits
- && row.1.group == group
- && row.1.m == m
- && row.1.shapes & mask != 0
- })
- .collect();
- assert_eq!(matches.len(), 1, "route coverage for {device_label} {format_name} M={m} {shape_name}");
- let (row_index, row) = matches[0];
- matched_rows[row_index] = true;
- let selected = row.route;
- canonical.push(format!("{device_label}|{format_name}|{shape_name}|{m}|{n}|{k}|{selected:?}"));
- let problem = problem(m, n, k, bits, group, prologue);
- assert_eq!(route(device_name, apple_gpu_family, supports_mxu, &problem, true), Some(selected));
- let runtime = MatmulMetalKernel::choose_dispatch(
- &problem,
- GemmBLayout::RowMajor,
- device_name,
- gpu_core_count,
- apple_gpu_family,
- supports_mxu,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- );
- if let QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) = selected {
- let specialization = GemvSpecialization::select_tile(
- &problem,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- tile,
- )
- .expect("stored GEMV tile must be legal");
- assert!(matches!(runtime, MatmulDispatch::Gemv(actual) if actual == specialization));
- } else if let QmvRoute::MainGemm(plan) = selected {
- assert!(matches!(runtime, MatmulDispatch::Gemm(actual) if actual == plan));
- }
- }
- }
- }
- }
- assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");
- canonical.sort();
- assert_eq!(canonical.len(), 462);
- assert_eq!(xxh3_64(canonical.join("\n").as_bytes()), FROZEN_PLAN_FINGERPRINT);
-}
-
-#[uzu_test]
-fn weaver_linears_select_the_tuned_tile() {
- for n in [5120, 6144] {
- let mut p = problem(4, n, 2048, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- p.params_layout = Some(QuantParamsLayout::GroupOutput);
- p.d_transform = GemmDTransform::RHT;
- let tile = GemvTile::quantized_output_tile(2, 16, 4, 8, 1);
- assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::Tuned(tile)));
- assert!(
- GemvSpecialization::select_tile(
- &p,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- tile
- )
- .is_some()
- );
- }
-}
-
-#[uzu_test]
-fn exact_lookup_rejects_non_matrix_inputs() {
- let p = problem(2, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- assert!(route("Apple M1", MTLGPUFamily::Apple7, false, &p, true).is_some());
- for mutate in [
- |p: &mut MatmulShape| p.m = 1,
- |p: &mut MatmulShape| p.n = 1,
- |p: &mut MatmulShape| p.b_bits = Some(8),
- |p: &mut MatmulShape| p.gathered = true,
- ] {
- let mut rejected = p;
- mutate(&mut rejected);
- assert!(route("Apple M1", MTLGPUFamily::Apple7, false, &rejected, true).is_none());
- }
- assert!(route("Apple M1", MTLGPUFamily::Apple7, false, &p, false).is_none());
-
- let mut rht = p;
- rht.d_transform = GemmDTransform::RHT;
- rht.signed_codes = true;
- let selected =
- route("Apple M1", MTLGPUFamily::Apple7, false, &rht, true).expect("RHT must preserve the exact route");
- let QmvRoute::Tuned(tile) = selected else {
- panic!("test anchor must use a tuned tile");
- };
- let deferred = GemvSpecialization::select_tile(
- &rht,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- tile,
- )
- .unwrap();
- assert_eq!(deferred.output_row_tile(), 16);
- assert!(!deferred.fuses_rht());
- rht.n -= 1;
- assert!(
- GemvSpecialization::select_tile(
- &rht,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- tile
- )
- .is_none()
- );
-
- let p = problem(7, 5120, 17408, 8, 64, GemmBPrologueKind::ScaleSymmetricDequant);
- assert!(matches!(route("Apple M5 Max", MTLGPUFamily::Apple10, true, &p, true), Some(QmvRoute::MainGemm(_))));
- assert_eq!(route("Apple M5 Max", MTLGPUFamily::Apple10, false, &p, true), None);
-}
-
-#[uzu_test]
-fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
- for (m, n, k) in [(1, 5120, 17408), (2, 4096, 5120)] {
- let problem = problem(m, n, k, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- assert_eq!(route("Apple M1", MTLGPUFamily::Apple7, false, &problem, true), None);
- let specialization = GemvSpecialization::select_shape(
- &problem,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- 8,
- MTLGPUFamily::Apple7,
- )
- .expect("normal M1 policy should select GEMV for this anchor");
- assert!(matches!(
- MatmulMetalKernel::choose_dispatch(
- &problem, GemmBLayout::RowMajor,
- "Apple M1",
- 8,
- MTLGPUFamily::Apple7,
- false,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- ),
- MatmulDispatch::Gemv(actual) if actual == specialization
- ));
- }
-
- let mut fp = problem(1, 1024, 512, 4, 32, GemmBPrologueKind::ScaleZeroPointDequant);
- fp.b_prologue = GemmBPrologueKind::FullPrecision;
- fp.b_bits = None;
- fp.b_group_size = None;
- let mut with_rht = fp;
- with_rht.d_transform = GemmDTransform::RHT;
- for (cores, family) in [(8, MTLGPUFamily::Apple7), (40, MTLGPUFamily::Apple10)] {
- let plain = GemvSpecialization::select_shape(
- &fp,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- cores,
- family,
- )
- .expect("generic M=1 FP GEMV tile");
- let rht = GemvSpecialization::select_shape(
- &with_rht,
- GemmBLayout::RowMajor,
- DataType::BF16,
- DataType::BF16,
- DataType::BF16,
- cores,
- family,
- )
- .expect("generic M=1 FP RHT GEMV tile");
- assert!(plain.output_row_tile() < 32);
- assert_eq!(rht, plain);
- }
-}
-
-#[uzu_test]
-fn family_lookup_requires_one_unanimous_route() {
- let m1_route = problem(4, 34816, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- let families: [(&MatmulShape, MTLGPUFamily, bool, &str, &[&str]); 4] = [
- (&m1_route, MTLGPUFamily::Apple7, false, "Apple M1", &["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"]),
- (&unanimous, MTLGPUFamily::Apple8, false, "Apple M2", &["Apple M2 Max", "Apple M2 Ultra"]),
- (&unanimous, MTLGPUFamily::Apple9, false, "Apple M3 Max", &["Apple M3", "Apple M3 Pro", "Apple M4 Max"]),
- (&unanimous, MTLGPUFamily::Apple10, true, "Apple M5 Max", &["Apple M5", "Apple M5 Pro"]),
- ];
- for (shape, family, supports_mxu, measured_device, aliases) in families {
- let measured = route(measured_device, family, supports_mxu, shape, true);
- for device in aliases {
- assert_eq!(route(device, family, supports_mxu, shape, true), measured);
- }
- }
-
- let disagreement = problem(3, 5120, 6144, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- assert_eq!(route("Apple M2 Max", MTLGPUFamily::Apple8, false, &disagreement, true), None);
- assert!(route("Apple M2", MTLGPUFamily::Apple8, false, &disagreement, true).is_some());
- assert_eq!(route("Apple A11 GPU", MTLGPUFamily::Apple4, false, &m1_route, true), None);
-}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
index eb5b606a5..bb707a29f 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/quant_dispatch_test.rs
@@ -12,24 +12,32 @@ use crate::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
- gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
+ gpu_types::{
+ activation_type::ActivationType, quantization::QuantizationMode,
+ quantization_method::QuantizationMethod,
+ },
kernel::{
- Kernels,
+ ActivationQuantization, ActivationTransform, GatedActMul, GatedActMulSettings, Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{MatmulDOps, MatmulKernel, MatmulShape, QuantParams, QuantParamsLayout, QuantParamsStrides},
+ matmul::{
+ ActivationFormat, Int8CodeLayout, MatmulA, MatmulArguments, MatmulKernel, MatmulShape, QuantParams,
+ QuantParamsLayout, QuantParamsStrides, packed_half_bytes,
+ },
},
},
- cpu::Cpu,
metal::{
Metal,
context::MetalContext,
- gpu_types::gemm::GemmBLayout,
- kernel::matmul::{MatmulMetalKernel, gemm::GemmEngine},
+ gpu_types::gemm::{GemmBLayout, GemmTiling},
+ kernel::matmul::{
+ GemmKernel, MatmulMetalKernel, MatmulOutputWork,
+ gemm::{GemmEngine, GemmPlan},
+ },
},
},
- data_type::DataTypeRepr,
+ data_type::{DataType, DataTypeRepr},
tests::{
- helpers::buffer_to_vec,
+ helpers::{buffer_to_vec, create_buffer, create_buffer_with_data},
matmul::{QuantBuffers, QuantInput, quant::run_quant_cpu, quant_arguments},
},
};
@@ -41,7 +49,15 @@ fn tile_weights<T: DataTypeRepr + Float>(
context: &MetalContext,
) {
let shape = MatmulShape {
- a_full_precision: false,
+ a: ActivationFormat::Int8(
+ ActivationQuantization::new(
+ ACTIVATION_SCALE_GROUP_SIZE,
+ input.group_size,
+ false,
+ Int8CodeLayout::for_right_bits(DataType::from(input.mode).size_in_bits() as u32).unwrap(),
+ )
+ .unwrap(),
+ ),
..MatmulShape::from_arguments(&quant_arguments(buffers, input))
};
buffers.weights_layout =
@@ -66,16 +82,6 @@ fn optimize_weights_layout_prepares_unsigned_codes() {
let signed_shape = MatmulShape::from_arguments(&quant_arguments(&mut expected, &signed));
let unsigned_shape = MatmulShape::from_arguments(&quant_arguments(&mut actual, &unsigned));
- let unsupported_shape = MatmulShape {
- a_full_precision: true,
- ..unsigned_shape
- };
- assert_eq!(
- kernel.optimize_weights_layout(&unsupported_shape, &mut actual.w, &context).unwrap(),
- (GemmBLayout::RowMajor, false),
- );
- assert_eq!(buffer_to_vec::<Metal, u32>(&context, &actual.w), unsigned.weights_for_upload());
-
assert_eq!(
kernel.optimize_weights_layout(&signed_shape, &mut expected.w, &context).unwrap(),
(GemmBLayout::MxuTiled, true),
@@ -94,13 +100,8 @@ fn run_quant_metal<T: DataTypeRepr + Float>(
dispatch: Option<GemmEngine>,
) -> Vec<T> {
let mut buffers = QuantBuffers::<Metal, T>::allocate(context, input);
- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
- context,
- T::data_type(),
- T::data_type(),
- T::data_type(),
- )
- .expect("MatmulMetalKernel");
+ let mut matmul =
+ MatmulMetalKernel::new(context, T::data_type(), T::data_type(), T::data_type()).expect("MatmulMetalKernel");
if gemm_b_layout == GemmBLayout::MxuTiled {
tile_weights(&matmul, &mut buffers, input, context);
}
@@ -115,17 +116,6 @@ fn run_quant_metal<T: DataTypeRepr + Float>(
buffer_to_vec::<Metal, T>(context, &buffers.y)
}
-fn check_tolerance(
- expected: f32,
- actual: f32,
- rel_tol: f64,
- abs_tol: f64,
-) -> bool {
- let diff = (expected - actual).abs() as f64;
- let tol = abs_tol.max(expected.abs() as f64 * rel_tol);
- diff <= tol
-}
-
#[uzu_test]
fn quant_params_layout_table() {
let cases = [
@@ -153,18 +143,13 @@ fn assert_parity<T: DataTypeRepr + Float + Debug + Display>(
abs_tol: f64,
) {
assert_eq!(reference.len(), actual.len(), "{label}: length mismatch");
- let mut errors = 0usize;
- for (i, (&exp, &got)) in reference.iter().zip(actual.iter()).enumerate() {
- let exp_f32 = exp.to_f32().unwrap();
- let got_f32 = got.to_f32().unwrap();
- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
- if errors < 5 {
- eprintln!(" {label} idx={i} expected={exp_f32} got={got_f32} diff={}", (exp_f32 - got_f32).abs());
- }
- errors += 1;
- }
+ for (i, (&expected, &actual)) in reference.iter().zip(actual).enumerate() {
+ let expected = expected.to_f32().unwrap();
+ let actual = actual.to_f32().unwrap();
+ let diff = f64::from((expected - actual).abs());
+ let tolerance = abs_tol.max(f64::from(expected.abs()) * rel_tol);
+ assert!(diff <= tolerance, "{label}[{i}]: expected {expected}, got {actual}, diff {diff} > {tolerance}");
}
- assert_eq!(errors, 0, "{label}: {errors} mismatches");
}
fn run_parity<T: DataTypeRepr + Float + Debug + Display>(
@@ -242,6 +227,8 @@ fn parity_bf16_8bit_splitk(
#[case::m4_gs32_4bit_zp(4, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::m8_gs32_4bit_zp(8, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::m8_gs32_4bit_mlx(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::m3_w4_g32_zp_k1024(3, 1024, 72, 32, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::m3_w8_g32_sym_k1024(3, 1024, 72, 32, 8, QuantizationMethod::ScaleSymmetric)]
fn parity_gemv_bf16(
#[case] m: u32,
#[case] k: u32,
@@ -256,81 +243,24 @@ fn parity_gemv_bf16(
#[uzu_test]
fn parity_bf16_gs32_4bit_mlx_with_bias() {
let context = block_on(MetalContext::new()).expect("Metal context");
- let input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
+ let mut input = QuantInput::<bf16>::new(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
- let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.5 + 0.1 * (j % 5) as f32).collect();
- let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+ input.output_bias = Some((0..input.n).map(|j| bf16::from_f32(0.5 + 0.1 * (j % 5) as f32)).collect());
- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let bias_pp_buf = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &bias_t);
- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("MatmulMetalKernel");
-
- let mut command_buffer = context.create_command_buffer(None, None).unwrap();
- let mut args = quant_arguments(&mut buffers, &input);
- args.output.ops = MatmulDOps {
- bias: Some(&bias_pp_buf),
- ..MatmulDOps::none()
- };
- matmul.encode_with_gemm_engine(args, GemmEngine::Simdgroup, &mut command_buffer).expect("encode quant with bias");
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);
-
- let mut reference = run_quant_cpu::<bf16>(&input);
- for i in 0..(input.m as usize) {
- for j in 0..(input.n as usize) {
- let idx = i * input.n as usize + j;
- reference[idx] = bf16::from_f32(reference[idx].to_f32() + bias_f32[j]);
- }
- }
+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, Some(GemmEngine::Simdgroup));
+ let reference = run_quant_cpu::<bf16>(&input);
assert_parity::<bf16>("with_bias", &reference, &actual, 0.05, 0.4);
}
#[uzu_test]
fn parity_bf16_gemv_qmv_fused_scale_bias() {
let context = block_on(MetalContext::new()).expect("Metal context");
- let input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
-
- let scale = 2.0_f32;
- let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.25 + 0.1 * (j % 7) as f32).collect();
- let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
-
- let raw = run_quant_cpu::<bf16>(&input);
- let reference: Vec<bf16> = (0..input.m as usize)
- .flat_map(|i| {
- (0..input.n as usize).map(move |j| {
- let idx = i * input.n as usize + j;
- (idx, j)
- })
- })
- .map(|(idx, j)| bf16::from_f32(scale * raw[idx].to_f32() + bias_f32[j]))
- .collect();
+ let mut input = QuantInput::<bf16>::new(1, 256, 64, 32, 4, QuantizationMethod::ScaleBias, 0);
- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let bias_buf = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &bias_t);
- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("MatmulMetalKernel");
-
- let mut command_buffer = context.create_command_buffer(None, None).unwrap();
- let mut args = quant_arguments(&mut buffers, &input);
- args.output.ops = MatmulDOps {
- ab_scale: scale,
- bias: Some(&bias_buf),
- ..MatmulDOps::none()
- };
- matmul.encode(args, &mut command_buffer).expect("encode quant gemv with scale+bias");
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);
+ input.ab_scale = 2.0;
+ input.output_bias = Some((0..input.n).map(|j| bf16::from_f32(0.25 + 0.1 * (j % 7) as f32)).collect());
+ let reference = run_quant_cpu::<bf16>(&input);
+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, None);
assert_parity::<bf16>("gemv_qmv_scale_bias", &reference, &actual, 0.05, 0.4);
}
@@ -396,66 +326,17 @@ fn parity_bf16_quant_rht(
let context = block_on(MetalContext::new()).expect("Metal context");
// W4/G64 matches the Qwen short-prefill route; keep the decode anchor at G32.
let input = QuantInput::<bf16>::new(m, k, n, group_size, 4, method, 0);
- let input = if group_size == 64 {
+ let mut input = if group_size == 64 {
input.with_group_output()
} else {
input
};
- let rht: Vec<i32> = (0..input.n as usize)
- .map(|i| {
- if i % 2 == 0 {
- 1
- } else {
- -1
- }
- })
- .collect();
- let bias_f32: Vec<f32> = (0..input.n as usize).map(|j| 0.3 + 0.05 * (j % 4) as f32).collect();
- let bias_t: Vec<bf16> = bias_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+ input.output_bias = with_bias.then(|| (0..input.n).map(|j| bf16::from_f32(0.3 + 0.05 * (j % 4) as f32)).collect());
+ input.output_rht_factors = Some((0..input.n).map(|i| 1 - 2 * (i % 2) as i32).collect());
// CPU reference applies the RHT through the CPU matmul kernel.
- let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("Cpu context");
- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);
- let cpu_rht = crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, &rht);
- let cpu_bias = crate::tests::helpers::create_buffer_with_data::<Cpu, bf16>(&cpu_context, &bias_t);
- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &cpu_context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("MatmulCpuKernel");
- let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("cpu command buffer");
- let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);
- cpu_args.output.ops = MatmulDOps {
- bias: with_bias.then_some(&cpu_bias),
- rht_factors: Some(&cpu_rht),
- ..MatmulDOps::none()
- };
- cpu_matmul.encode(cpu_args, &mut cpu_command_buffer).expect("cpu encode quant+rht");
- block_on(cpu_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let reference = buffer_to_vec::<Cpu, bf16>(&cpu_context, &cpu_buffers.y);
-
- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let metal_rht = crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, &rht);
- let metal_bias = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &bias_t);
- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("MatmulMetalKernel");
- let mut command_buffer = context.create_command_buffer(None, None).unwrap();
- let mut args = quant_arguments(&mut buffers, &input);
- args.output.ops = MatmulDOps {
- bias: with_bias.then_some(&metal_bias),
- rht_factors: Some(&metal_rht),
- ..MatmulDOps::none()
- };
- matmul.encode(args, &mut command_buffer).expect("encode quant matmul with rht");
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);
+ let reference = run_quant_cpu::<bf16>(&input);
+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, None);
assert_parity::<bf16>(
if with_bias {
@@ -667,21 +548,24 @@ fn a8w4_zero_point_tail_parity(#[case] m: u32) {
#[rstest]
#[test_attr(uzu_test)]
-#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256u32)]
-#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256u32)]
-#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256u32)]
+#[case::w4_bias(4, QuantizationMethod::ScaleBias, 256, 2, 32)]
+#[case::w4_zp(4, QuantizationMethod::ScaleZeroPoint, 256, 2, 32)]
+#[case::w8_sym(8, QuantizationMethod::ScaleSymmetric, 256, 2, 32)]
+#[case::direct_w8_m5(8, QuantizationMethod::ScaleSymmetric, 1024, 5, 64)]
fn signed_weights_full_precision_activations_parity_bf16(
#[case] bits: u32,
#[case] method: QuantizationMethod,
#[case] k: u32,
+ #[case] m: u32,
+ #[case] group_size: u32,
) {
let context = block_on(MetalContext::new()).expect("Metal context");
- let (m, n, group_size) = (2u32, 128u32, 32u32);
+ let n = 128;
let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0).with_signed_weight_codes();
let reference_input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0);
let reference = run_quant_cpu::<bf16>(&reference_input);
- for (label, dispatch) in [("gemv", None), ("simdgroup", Some(GemmEngine::Simdgroup))] {
+ for (label, dispatch) in [("auto", None), ("simdgroup", Some(GemmEngine::Simdgroup))] {
let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, dispatch);
assert_parity::<bf16>(
&format!("signed weights FP-A {label} bits={bits} method={method:?}"),
@@ -715,9 +599,10 @@ fn a8w_mxu_output_bias_parity_bf16(
72u32
},
);
- let (input, reference_input) = QuantInput::<bf16>::new(m, k, n, 32, bits, QuantizationMethod::ScaleSymmetric, 0)
- .with_group_output()
- .with_prepared_a_and_reference(ACTIVATION_SCALE_GROUP_SIZE, None);
+ let (mut input, mut reference_input) =
+ QuantInput::<bf16>::new(m, k, n, 32, bits, QuantizationMethod::ScaleSymmetric, 0)
+ .with_group_output()
+ .with_prepared_a_and_reference(ACTIVATION_SCALE_GROUP_SIZE, None);
let output_bias: Vec<bf16> = (0..n).map(|column| bf16::from_f32(0.25 + 0.05 * (column % 7) as f32)).collect();
let output_hadamard_factors: Option<Vec<i32>> = with_output_hadamard.then(|| {
(0..n)
@@ -730,58 +615,14 @@ fn a8w_mxu_output_bias_parity_bf16(
})
.collect()
});
+ input.output_bias = Some(output_bias.clone());
+ reference_input.output_bias = Some(output_bias);
+ input.output_rht_factors = output_hadamard_factors.clone();
+ reference_input.output_rht_factors = output_hadamard_factors;
- let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("CPU context");
- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &reference_input);
- let cpu_output_hadamard_factors = output_hadamard_factors
- .as_ref()
- .map(|factors| crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, factors));
- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &cpu_context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("CPU matmul kernel");
- let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");
- let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &reference_input);
- cpu_arguments.output.ops = MatmulDOps {
- rht_factors: cpu_output_hadamard_factors.as_ref(),
- ..MatmulDOps::none()
- };
- cpu_matmul.encode(cpu_arguments, &mut cpu_command_buffer).expect("CPU A8W matmul");
- block_on(cpu_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let mut reference = buffer_to_vec::<Cpu, bf16>(&cpu_context, &cpu_buffers.y);
- for row in reference.chunks_exact_mut(n as usize) {
- for (value, bias) in row.iter_mut().zip(&output_bias) {
- *value = bf16::from_f32(value.to_f32() + bias.to_f32());
- }
- }
+ let reference = run_quant_cpu::<bf16>(&reference_input);
- let mut metal_buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let metal_output_bias = crate::tests::helpers::create_buffer_with_data::<Metal, bf16>(&context, &output_bias);
- let metal_output_hadamard_factors = output_hadamard_factors
- .as_ref()
- .map(|factors| crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, factors));
- let mut metal_matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("Metal matmul kernel");
- let mut metal_command_buffer = context.create_command_buffer(None, None).unwrap();
- let mut metal_arguments = quant_arguments(&mut metal_buffers, &input);
- metal_arguments.output.ops = MatmulDOps {
- bias: Some(&metal_output_bias),
- rht_factors: metal_output_hadamard_factors.as_ref(),
- ..MatmulDOps::none()
- };
- metal_matmul
- .encode_with_gemm_engine(metal_arguments, GemmEngine::Mxu, &mut metal_command_buffer)
- .expect("Metal A8W MXU matmul with output bias");
- block_on(metal_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let actual = buffer_to_vec::<Metal, bf16>(&context, &metal_buffers.y);
+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::RowMajor, Some(GemmEngine::Mxu));
assert_parity::<bf16>(
&format!("A8W{bits} MXU output bias, output Hadamard={with_output_hadamard}"),
@@ -808,13 +649,14 @@ fn gemv_reads_mxu_tiled_layout_bf16(
#[case] bits: u32,
#[case] method: QuantizationMethod,
#[values(false, true)] output_hadamard: bool,
+ #[values(1, 2, 3, 4)] m: u32,
) {
let context = block_on(MetalContext::new()).expect("Metal context");
if !context.supports_mxu {
return;
}
- let input = QuantInput::<bf16>::new(1, k, n, group_size, bits, method, 0).with_group_output();
- let input = if bits == 8 {
+ let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0).with_group_output();
+ let mut input = if bits == 8 {
input.with_signed_weight_codes()
} else {
input
@@ -828,49 +670,15 @@ fn gemv_reads_mxu_tiled_layout_bf16(
}
})
.collect();
+ input.output_rht_factors = output_hadamard.then_some(factors);
- let cpu_context = block_on(<Cpu as Backend>::Context::new()).expect("CPU context");
- let mut cpu_buffers = QuantBuffers::<Cpu, bf16>::allocate(&cpu_context, &input);
- let cpu_factors = crate::tests::helpers::create_buffer_with_data::<Cpu, i32>(&cpu_context, &factors);
- let mut cpu_matmul = <<Cpu as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &cpu_context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("CPU matmul kernel");
- let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");
- let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &input);
- cpu_arguments.output.ops = MatmulDOps {
- rht_factors: output_hadamard.then_some(&cpu_factors),
- ..MatmulDOps::none()
- };
- cpu_matmul.encode(cpu_arguments, &mut cpu_command_buffer).expect("CPU matmul");
- block_on(cpu_command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let reference = buffer_to_vec::<Cpu, bf16>(&cpu_context, &cpu_buffers.y);
-
- let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
- let metal_factors = crate::tests::helpers::create_buffer_with_data::<Metal, i32>(&context, &factors);
- let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
- &context,
- bf16::data_type(),
- bf16::data_type(),
- bf16::data_type(),
- )
- .expect("Metal matmul kernel");
- tile_weights(&matmul, &mut buffers, &input, &context);
- let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
- let mut arguments = quant_arguments(&mut buffers, &input);
- arguments.output.ops = MatmulDOps {
- rht_factors: output_hadamard.then_some(&metal_factors),
- ..MatmulDOps::none()
- };
- matmul.encode(arguments, &mut command_buffer).expect("Metal matmul");
- block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
- let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);
+ let reference = run_quant_cpu::<bf16>(&input);
+ let actual = run_quant_metal::<bf16>(&context, &input, GemmBLayout::MxuTiled, None);
assert_parity::<bf16>(
- &format!("MxuTiled GEMV k={k} n={n} gs={group_size} bits={bits} method={method:?} hadamard={output_hadamard}"),
+ &format!(
+ "MxuTiled GEMV m={m} k={k} n={n} gs={group_size} bits={bits} method={method:?} hadamard={output_hadamard}"
+ ),
&reference,
&actual,
0.05,
@@ -928,3 +736,115 @@ fn gathered_gemv_reads_mxu_tiled_layout_bf16(
0.5,
);
}
+
+fn run_packed_half_pipeline(
+ group_size: u32,
+ gated: bool,
+ (batch, n): (u32, u32),
+ with_bias: bool,
+ with_rht: bool,
+ soft_cap: Option<f32>,
+ method: QuantizationMethod,
+ split_k: u32,
+) {
+ let k = 512;
+ let source_width = k * (1 + gated as u32);
+ let source: Vec<bf16> = (0..batch * source_width).map(|i| bf16::from_f32((i as f32 * 0.013).sin() * 0.2)).collect();
+ let signs = |len: u32| (0..len).map(|i| 1 - 2 * (i % 3 == 0) as i32).collect::<Vec<_>>();
+ let factors = signs(k);
+ let format = ActivationFormat::PackedHalf {
+ group_size,
+ };
+ let mut input = QuantInput::<bf16>::new(batch, k, n, group_size, 4, method, 19).with_group_output();
+ input.ab_scale = 0.75;
+ input.output_bias = with_bias.then(|| (0..n).map(|i| bf16::from_f32(1.0 + 0.5 * (i % 3) as f32)).collect());
+ input.output_rht_factors = with_rht.then(|| signs(n));
+ input.soft_cap = soft_cap;
+ let context = block_on(MetalContext::new()).expect("Metal context");
+ let mut buffers = QuantBuffers::<Metal, bf16>::allocate(&context, &input);
+ let source_buffer = create_buffer_with_data::<Metal, bf16>(&context, &source);
+ let factors_buffer = create_buffer_with_data::<Metal, i32>(&context, &factors);
+ let mut reference_a = create_buffer::<Metal, bf16>(&context, (batch * k) as usize);
+ let mut gemm = GemmKernel::new(DataType::BF16, DataType::BF16, DataType::BF16);
+ let output_work = MatmulOutputWork::new(&context, DataType::BF16, DataType::BF16).unwrap();
+ let mut packed = create_buffer::<Metal, u32>(&context, packed_half_bytes(k, group_size) / size_of::<u32>());
+ let mut command_buffer = context.create_command_buffer(None, None).expect("Metal command buffer");
+ if !gated {
+ ActivationTransform::<Metal>::input_rht(&context, DataType::BF16, false).expect("RHT reference").encode_fp(
+ &source_buffer,
+ &mut reference_a,
+ &factors_buffer,
+ batch,
+ k,
+ &mut command_buffer,
+ );
+ ActivationTransform::<Metal>::quantize(&context, DataType::BF16, format)
+ .expect("Metal RHT PackedHalf pack")
+ .encode_pack_half(&source_buffer, &mut packed, &factors_buffer, batch, k, &mut command_buffer);
+ } else {
+ GatedActMul::<Metal>::full_precision(&context, DataType::BF16, true, true, GatedActMulSettings::default())
+ .expect("gate reference")
+ .encode_fp(
+ &source_buffer,
+ None::<&<Metal as Backend>::GlobalGpuBuffer>,
+ &mut reference_a,
+ Some(&factors_buffer),
+ k,
+ batch,
+ 0,
+ 0,
+ ActivationType::SILU,
+ &mut command_buffer,
+ );
+ GatedActMul::<Metal>::quantized(&context, DataType::BF16, format, GatedActMulSettings::default())
+ .expect("Metal PackedHalf gated activation")
+ .encode_pack_half(
+ &source_buffer,
+ &mut packed,
+ &factors_buffer,
+ k,
+ batch,
+ ActivationType::SILU,
+ &mut command_buffer,
+ );
+ }
+ let arguments = quant_arguments(&mut buffers, &input);
+ let arguments = MatmulArguments {
+ a: MatmulA::PackedHalf {
+ values: &packed,
+ group_size,
+ },
+ b: arguments.b,
+ b_leading_dimension: arguments.b_leading_dimension,
+ b_transpose: arguments.b_transpose,
+ output: arguments.output,
+ gather_indices: arguments.gather_indices,
+ m: arguments.m,
+ n: arguments.n,
+ k: arguments.k,
+ };
+ let plan = GemmPlan {
+ engine: GemmEngine::Simdgroup,
+ tiling: GemmTiling::Tile8x32x64_Simdgroups1x2x4,
+ split_k,
+ };
+ gemm.encode_plan(arguments, plan, &output_work, &mut command_buffer).expect("prepared W4 GEMM");
+ block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
+ input.x = buffer_to_vec::<Metal, bf16>(&context, &reference_a);
+ let reference = run_quant_cpu::<bf16>(&input);
+ let actual = buffer_to_vec::<Metal, bf16>(&context, &buffers.y);
+ assert_parity::<bf16>(&format!("gated={gated} G{group_size} M{batch} K{k}"), &reference, &actual, 0.05, 0.4);
+}
+
+#[uzu_test]
+fn packed_half_producers_match_bf16_reference() {
+ // Fused RHT rounds directly to half; the standalone reference stores BF16 first.
+ for (group, gated, n, bias, rht, cap, method, splits) in [
+ (32, false, (4, 72), false, false, None, ScaleZeroPoint, 2),
+ (64, false, (5, 64), true, false, None, ScaleBias, 1),
+ (32, true, (8, 64), true, true, None, ScaleZeroPoint, 1),
+ (64, true, (5, 64), true, false, Some(0.5), ScaleBias, 2),
+ ] {
+ run_packed_half_pipeline(group, gated, n, bias, rht, cap, method, splits);
+ }
+}
diff --git a/crates/uzu-engine/unit/common/matmul/mod.rs b/crates/uzu-engine/unit/common/matmul/mod.rs
index 7d844fd12..1112d03e4 100644
--- a/crates/uzu-engine/unit/common/matmul/mod.rs
+++ b/crates/uzu-engine/unit/common/matmul/mod.rs
@@ -7,6 +7,6 @@ pub use bench::{iter_encode_loop, iter_encode_loop_named};
pub use harness::{Case, cpu_reference, deterministic_input};
pub use quant::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu, transpose_metadata};
pub use shape::{
- Shape, all_correctness_shapes, bench_fp_gemm_shapes, bench_quant_gemm_shapes, bench_quant_gemv_shapes,
- qwen3_layer_shapes,
+ Shape, a8_format, all_correctness_shapes, bench_fp_gemm_shapes, bench_quant_gemm_shapes, bench_quant_gemv_shapes,
+ matmul_shape, quantized_matmul_shape, qwen3_layer_shapes,
};
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index 6b921832e..88984a91e 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -10,7 +10,7 @@ use crate::{
Backend, BufferRef, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
gpu_types::{quantization::QuantizationMode, quantization_method::QuantizationMethod},
kernel::{
- ActivationQuantization, ActivationTransform, Kernels,
+ ActivationFormat, ActivationQuantization, ActivationTransform, Kernels,
matmul::{
Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,
QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection, WeightsLayout,
@@ -38,6 +38,10 @@ pub struct QuantInput<T: DataTypeRepr + Float> {
pub zero_points: Option<Vec<u8>>,
pub biases: Option<Vec<T>>,
pub x: Vec<T>,
+ pub ab_scale: f32,
+ pub soft_cap: Option<f32>,
+ pub output_bias: Option<Vec<T>>,
+ pub output_rht_factors: Option<Vec<i32>>,
pub k: u32,
pub n: u32,
pub m: u32,
@@ -165,6 +169,10 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {
zero_points,
biases,
x,
+ ab_scale: 1.0,
+ soft_cap: None,
+ output_bias: None,
+ output_rht_factors: None,
k,
n,
m,
@@ -241,8 +249,9 @@ impl<T: DataTypeRepr + Float> QuantInput<T> {
code_layout,
)
.expect("supported activation quantization");
- let transform = ActivationTransform::<Cpu>::quantize(&context, T::data_type(), quantization)
- .expect("CPU activation quantization transform");
+ let transform =
+ ActivationTransform::<Cpu>::quantize(&context, T::data_type(), ActivationFormat::Int8(quantization))
+ .expect("CPU activation quantization transform");
let mut command_buffer = context.create_command_buffer(None, None).expect("CPU command buffer");
transform.encode_quantize(
&input,
@@ -292,6 +301,8 @@ pub struct QuantBuffers<B: Backend, T: DataTypeRepr + Float> {
pub prepared_a: Option<B::GlobalGpuBuffer>,
pub prepared_a_scales: Option<B::GlobalGpuBuffer>,
pub prepared_a_group_sums: Option<B::GlobalGpuBuffer>,
+ pub output_bias: Option<B::GlobalGpuBuffer>,
+ pub output_rht_factors: Option<B::GlobalGpuBuffer>,
pub y: B::GlobalGpuBuffer,
pub weights_layout: WeightsLayout<B>,
_t: std::marker::PhantomData<T>,
@@ -339,6 +350,11 @@ impl<B: Backend, T: DataTypeRepr + Float> QuantBuffers<B, T> {
.as_ref()
.filter(|prepared| !prepared.group_sums.is_empty())
.map(|prepared| create_buffer_with_data::<B, i32>(context, &prepared.group_sums)),
+ output_bias: input.output_bias.as_ref().map(|bias| create_buffer_with_data::<B, T>(context, bias)),
+ output_rht_factors: input
+ .output_rht_factors
+ .as_ref()
+ .map(|factors| create_buffer_with_data::<B, i32>(context, factors)),
y: create_buffer::<B, T>(context, (input.m as usize) * (input.n as usize)),
weights_layout: Default::default(),
_t: std::marker::PhantomData,
@@ -408,6 +424,8 @@ pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(
prepared_a,
prepared_a_scales,
prepared_a_group_sums,
+ output_bias,
+ output_rht_factors,
y,
weights_layout,
..
@@ -420,20 +438,24 @@ pub fn quant_arguments<'a, B: Backend, T: DataTypeRepr + Float>(
// Symmetric weights carry no correction term, so the GEMM never reads these.
group_sums: (input.quant_method != QuantizationMethod::ScaleSymmetric)
.then(|| prepared_a_group_sums.as_ref().expect("prepared activation row sums")),
- scale_group_size: prepared.quantization.scale_group_size(),
- code_layout: prepared.quantization.code_layout(),
- },
- None => MatmulA::FullPrecision {
- values: &*x,
- offset: 0,
+ quantization: prepared.quantization,
},
+ None => MatmulA::full_precision(&*x),
};
MatmulArguments {
a,
b,
b_leading_dimension: None,
b_transpose: true,
- output: MatmulOutput::new(y, MatmulDOps::none()),
+ output: MatmulOutput::new(
+ y,
+ MatmulDOps {
+ ab_scale: input.ab_scale,
+ bias: output_bias.as_ref(),
+ rht_factors: output_rht_factors.as_ref(),
+ soft_cap: input.soft_cap,
+ },
+ ),
gather_indices: None,
m: input.m,
n: input.n,
diff --git a/crates/uzu-engine/unit/common/matmul/shape.rs b/crates/uzu-engine/unit/common/matmul/shape.rs
index 9ee092cf1..970b25329 100644
--- a/crates/uzu-engine/unit/common/matmul/shape.rs
+++ b/crates/uzu-engine/unit/common/matmul/shape.rs
@@ -1,5 +1,13 @@
use derive_more::Display;
+use crate::backends::common::{
+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},
+ kernel::{
+ ActivationQuantization,
+ matmul::{ActivationFormat, Int8CodeLayout, MatmulShape, QuantParamsLayout},
+ },
+};
+
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
#[display("M[{m}]K[{k}]N[{n}]")]
pub struct Shape {
@@ -111,3 +119,51 @@ pub fn qwen3_layer_shapes(bits: u32) -> impl Iterator<Item = (&'static str, Shap
.filter(move |&&(_, k, _)| k.is_multiple_of(block_size))
.flat_map(move |&(label, k, n)| ms.iter().map(move |&m| (label, Shape::new(m, k, n))))
}
+
+pub fn matmul_shape(
+ m: u32,
+ n: u32,
+ k: u32,
+) -> MatmulShape {
+ MatmulShape {
+ m,
+ n,
+ k,
+ b_transpose: true,
+ b_leading_dimension: None,
+ b_prologue: GemmBPrologueKind::FullPrecision,
+ b_is_trellis: false,
+ b_bits: None,
+ b_group_size: None,
+ signed_codes: false,
+ a: ActivationFormat::Bf16,
+ gathered: false,
+ params_layout: Some(QuantParamsLayout::OutputGroup),
+ d_transform: GemmDTransform::empty(),
+ }
+}
+
+pub fn quantized_matmul_shape(
+ m: u32,
+ n: u32,
+ k: u32,
+ bits: u32,
+ group: u32,
+) -> MatmulShape {
+ MatmulShape {
+ b_prologue: if bits == 8 {
+ GemmBPrologueKind::ScaleSymmetricDequant
+ } else {
+ GemmBPrologueKind::ScaleZeroPointDequant
+ },
+ b_bits: Some(bits),
+ b_group_size: Some(group),
+ signed_codes: bits == 8,
+ params_layout: Some(QuantParamsLayout::GroupOutput),
+ ..matmul_shape(m, n, k)
+ }
+}
+
+pub fn a8_format() -> ActivationFormat {
+ ActivationFormat::Int8(ActivationQuantization::new(128, 128, false, Int8CodeLayout::GroupedByNibble).unwrap())
+}