A high-performance inference engine for AI models
Signed Quantized Weights (#649)
eugene4978ea27a1f0
diff --git a/crates/backend-uzu/BENCHMARKS.md b/crates/backend-uzu/BENCHMARKS.mdindex 087c12bac..771f4860b 100644--- a/crates/backend-uzu/BENCHMARKS.md+++ b/crates/backend-uzu/BENCHMARKS.md@@ -32,6 +32,7 @@ target is `--lib`.|-----------------------------------------|-------------------------------------|| `Metal/Kernel/Matmul/GEMM` | `Metal/Kernel/Matmul/GEMM` || `Metal/Kernel/Matmul/GEMM_MXU` | `Metal/Kernel/Matmul/GEMM_MXU` |+| `Metal/Kernel/A8W/w4`, `.../w8` | `Metal/Kernel/A8W` || `Metal/Kernel/UnifiedQuantizedGemm/...` | `Metal/Kernel/UnifiedQuantizedGemm` || `Metal/Kernel/Gemv/...` | `Metal/Kernel/Gemv` || `Metal/Kernel/Qwen3Layers/...` | `Metal/Kernel/Qwen3Layers` |@@ -72,31 +73,39 @@ directory.Run one benchmark group at a time to avoid the iOS watchdog killing theapp.+Set `IPHONEOS_DEPLOYMENT_TARGET` (value from `platforms.toml` `[envs]`)+for all iOS builds; without it the link step fails with undefined+symbols (e.g. `___chkstk_darwin`) because objects are built for a newer+SDK than the default deployment target.+Key flags:-- `-e CRITERION_HOME=target/criterion/a19` — on-device env var. Path is+- `-e CRITERION_HOME=criterion/a19` — on-device env var. Path isrelative to the app's cwd (`Documents/`), so this becomes- `Documents/target/criterion/a19/` on device.-- `--copy-back "Documents/target=$(pwd)/target"` — after the run,- `cargo-dinghy` pulls `Documents/target` from the device into your- repo's `target/`. `$(pwd)` is required (absolute DST) because the+ `Documents/criterion/a19/` on device. Keep it directly under+ `Documents/` — nested parents (e.g. `Documents/target/`) do not exist+ on a fresh install and the pre-run sync cannot create them.+- `--sync-dirs "$(pwd)/target/criterion=Documents/criterion"` —+ syncs the criterion tree between host and device before and after the+ run, so results written on device land back in the repo's+ `target/criterion/`. `$(pwd)` is required (absolute path) because thecargo runner is launched with cwd set to the package dir, not theworkspace root.```bashDEVICE=<DEVICE_ID>-cargo dinghy \+IPHONEOS_DEPLOYMENT_TARGET=26.4 cargo dinghy \-d "$DEVICE" \- -e CRITERION_HOME=target/criterion/a19 \- --copy-back "Documents/target=$(pwd)/target" \+ -e CRITERION_HOME=criterion/a19 \+ --sync-dirs "$(pwd)/target/criterion=Documents/criterion" \bench -p backend-uzu --lib -- \"Metal/Kernel/Matmul" \--save-baseline matmul_baseline_a19```-After the run completes you'll have-`target/criterion/a19/Metal/Kernel/Matmul/<GEMM|GEMM_MXU>/…/matmul_baseline_a19/`+On-device criterion sanitizes group path separators, so results land in+`target/criterion/a19/Metal_Kernel_Matmul_<GEMM|GEMM_MXU>/…/`on the host, next to any `m2_max/` baselines.## Viewing reportsdiff --git a/crates/backend-uzu/src/backends/common/gpu_types/activation_transform.rs b/crates/backend-uzu/src/backends/common/gpu_types/activation_transform.rsnew file mode 100644index 000000000..0e28f0315--- /dev/null+++ b/crates/backend-uzu/src/backends/common/gpu_types/activation_transform.rs@@ -0,0 +1,8 @@+#[repr(C)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum ActivationTransformOp {+ InputRht,+ OutputRht,+ Quantize,+ QuantizeWithGroupSums,+}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rsindex 0da9fa36f..429450dac 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/hadamard_order.rs@@ -1,8 +1 @@pub const HADAMARD_TRANSFORM_BLOCK_SIZE: usize = 32;--#[repr(C)]-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub enum HadamardTransformOrder {- Input,- Output,-}diff --git a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs b/crates/backend-uzu/src/backends/common/gpu_types/mod.rsindex ddfd65d65..d595972e0 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/mod.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/mod.rs@@ -3,6 +3,7 @@//! These `#[repr(C)]` structs are the source of truth. The build system uses//! cbindgen to generate C headers for Metal shaders.+pub mod activation_transform;pub mod activation_type;pub mod argmax;pub mod attention;@@ -16,6 +17,7 @@ pub mod ring;pub mod trie;pub mod weaver;+pub use activation_transform::*;pub use activation_type::*;pub use argmax::*;pub use attention::*;diff --git a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rsindex 8c15c8ca2..fce644746 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/quantization.rs@@ -29,6 +29,14 @@ impl QuantizationMode {QuantizationMode::U8 => DataType::U8,}}++ pub fn weight_codes_sign_flip_mask(&self) -> Option<u8> {+ match self {+ QuantizationMode::U4 => Some(0x88),+ QuantizationMode::I8 => None,+ QuantizationMode::U8 => Some(0x80),+ }+ }}impl From<QuantizationMode> for DataType {diff --git a/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs b/crates/backend-uzu/src/backends/common/kernel/activation_transform.rsnew file mode 100644index 000000000..e875902b6--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/activation_transform.rs@@ -0,0 +1,148 @@+use crate::{+ backends::common::{+ Allocation, Backend, Encoder, Kernels,+ gpu_types::{ActivationTransformOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+ kernel::ActivationTransformKernel,+ },+ data_type::DataType,+};++fn assert_row_width(element_count: u32) {+ assert!(+ (element_count as usize).is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),+ "activation transform requires element_count ({element_count}) to be a multiple of {HADAMARD_TRANSFORM_BLOCK_SIZE}"+ );+}++pub struct ActivationTransform<B: Backend> {+ kernel: <B::Kernels as Kernels>::ActivationTransformKernel,+ ops: ActivationTransformOp,+ in_place: bool,+}++impl<B: Backend> ActivationTransform<B> {+ fn new(+ context: &B::Context,+ data_type: DataType,+ ops: ActivationTransformOp,+ in_place: bool,+ ) -> Result<Self, B::Error> {+ let kernel = <B::Kernels as Kernels>::ActivationTransformKernel::new(context, data_type, ops, in_place)?;+ Ok(Self {+ kernel,+ ops,+ in_place,+ })+ }++ pub fn input_rht(+ context: &B::Context,+ data_type: DataType,+ in_place: bool,+ ) -> Result<Self, B::Error> {+ Self::new(context, data_type, ActivationTransformOp::InputRht, in_place)+ }++ pub fn output_rht(+ context: &B::Context,+ data_type: DataType,+ in_place: bool,+ ) -> Result<Self, B::Error> {+ Self::new(context, data_type, ActivationTransformOp::OutputRht, in_place)+ }++ pub fn quantize(+ context: &B::Context,+ data_type: DataType,+ emit_group_sums: bool,+ ) -> Result<Self, B::Error> {+ let ops = if emit_group_sums {+ ActivationTransformOp::QuantizeWithGroupSums+ } else {+ ActivationTransformOp::Quantize+ };+ Self::new(context, data_type, ops, false)+ }++ /// `input` and `output` must be distinct buffers.+ pub fn encode_fp(+ &self,+ input: &Allocation<B>,+ output: &mut Allocation<B>,+ rht_factors: &Allocation<B>,+ batch_size: u32,+ element_count: u32,+ encoder: &mut Encoder<B>,+ ) {+ assert!(!self.quantizes() && !self.in_place);+ assert_row_width(element_count);+ self.kernel.encode(+ Some(input),+ Some(output),+ None::<&mut Allocation<B>>,+ None::<&mut Allocation<B>>,+ None::<&mut Allocation<B>>,+ rht_factors,+ batch_size,+ element_count,+ encoder,+ );+ }++ pub fn encode_fp_in_place(+ &self,+ data: &mut Allocation<B>,+ rht_factors: &Allocation<B>,+ batch_size: u32,+ element_count: u32,+ encoder: &mut Encoder<B>,+ ) {+ assert!(!self.quantizes() && self.in_place);+ assert_row_width(element_count);+ self.kernel.encode(+ None::<&Allocation<B>>,+ Some(data),+ None::<&mut Allocation<B>>,+ None::<&mut Allocation<B>>,+ None::<&mut Allocation<B>>,+ rht_factors,+ batch_size,+ element_count,+ encoder,+ );+ }++ pub fn encode_quantize(+ &self,+ input: &Allocation<B>,+ q_out: &mut Allocation<B>,+ scales_out: &mut Allocation<B>,+ group_sums_out: Option<&mut Allocation<B>>,+ rht_factors: &Allocation<B>,+ batch_size: u32,+ element_count: u32,+ encoder: &mut Encoder<B>,+ ) {+ assert!(self.quantizes());+ assert_row_width(element_count);+ self.kernel.encode(+ Some(input),+ None::<&mut Allocation<B>>,+ Some(q_out),+ Some(scales_out),+ group_sums_out,+ rht_factors,+ batch_size,+ element_count,+ encoder,+ );+ }++ fn quantizes(&self) -> bool {+ matches!(self.ops, ActivationTransformOp::Quantize | ActivationTransformOp::QuantizeWithGroupSums)+ }++ pub fn emit_group_sums(&self) -> bool {+ self.ops == ActivationTransformOp::QuantizeWithGroupSums+ }+}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rsindex 0c4a6359f..de49d1364 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/kernel.rs@@ -1,5 +1,11 @@use crate::{- backends::common::{Backend, BufferArg, Encoder, Kernels, kernel::matmul::arguments::MatmulArguments},+ backends::common::{+ Backend, BufferArg, Encoder, Kernels,+ kernel::matmul::{+ arguments::MatmulArguments,+ routing::{MatmulPath, MatmulShape},+ },+ },data_type::DataType,};@@ -18,4 +24,12 @@ pub trait MatmulKernel: Sized + Send + Sync {arguments: MatmulArguments<'a, 'b, 'd, Self::Backend, TB>,encoder: &mut Encoder<Self::Backend>,) -> Result<(), <Self::Backend as Backend>::Error>;++ fn select_path(+ &self,+ _shape: &MatmulShape,+ _context: &<Self::Backend as Backend>::Context,+ ) -> MatmulPath {+ MatmulPath::Gemm+ }}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rsindex 6efb603ba..e472a5477 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs@@ -16,6 +16,7 @@ pub enum MatmulB<'a, B: Backend, TB: BufferArg<'a, B> = &'a Allocation<B>> {biases: &'a Allocation<B>,mode: QuantizationMode,group_size: u32,+ signed_codes: bool,},ScaleZeroPointDequant {b: &'a Allocation<B>,@@ -23,12 +24,14 @@ pub enum MatmulB<'a, B: Backend, TB: BufferArg<'a, B> = &'a Allocation<B>> {zero_points: &'a Allocation<B>,mode: QuantizationMode,group_size: u32,+ signed_codes: bool,},ScaleSymmetricDequant {b: &'a Allocation<B>,scales: &'a Allocation<B>,mode: QuantizationMode,group_size: u32,+ signed_codes: bool,},}@@ -89,4 +92,24 @@ impl<'a, B: Backend, TB: BufferArg<'a, B>> MatmulB<'a, B, TB> {} => Some(*group_size),}}++ pub fn signed_codes(&self) -> bool {+ match self {+ Self::FullPrecision {+ ..+ } => false,+ Self::ScaleBiasDequant {+ signed_codes,+ ..+ }+ | Self::ScaleZeroPointDequant {+ signed_codes,+ ..+ }+ | Self::ScaleSymmetricDequant {+ signed_codes,+ ..+ } => *signed_codes,+ }+ }}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rsindex 7e8d91981..c727ec22f 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/mod.rs@@ -4,6 +4,7 @@ mod error;mod kernel;mod matmul_a;mod matmul_b;+pub mod routing;pub use arguments::MatmulArguments;pub use d_ops::MatmulDOps;@@ -11,3 +12,4 @@ pub use error::MatmulError;pub use kernel::MatmulKernel;pub use matmul_a::MatmulA;pub use matmul_b::MatmulB;+pub use routing::{MatmulPath, MatmulShape};diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rsnew file mode 100644index 000000000..705c03573--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/routing.rs@@ -0,0 +1,52 @@+use super::{MatmulA, MatmulArguments};+use crate::backends::common::{+ Backend, BufferArg,+ gpu_types::gemm::{GemmBPrologueKind, GemmDTransform},+};++#[derive(Clone, Copy)]+pub struct MatmulShape {+ pub m: u32,+ pub n: u32,+ pub k: u32,+ pub b_transpose: bool,+ pub b_leading_dimension: Option<u32>,+ pub b_prologue: GemmBPrologueKind,+ pub b_bits: Option<u32>,+ pub b_group_size: Option<u32>,+ pub signed_codes: bool,+ pub a_full_precision: bool,+ pub gathered: bool,+ pub d_transform: GemmDTransform,+}++impl MatmulShape {+ pub fn from_arguments<'a, 'b, 'd, B: Backend, TB: BufferArg<'b, B>>(+ arguments: &MatmulArguments<'a, 'b, 'd, B, TB>+ ) -> Self {+ Self {+ m: arguments.m,+ n: arguments.n,+ k: arguments.k,+ b_transpose: arguments.b_transpose,+ b_leading_dimension: arguments.b_leading_dimension,+ b_prologue: arguments.b.b_prologue(),+ b_bits: arguments.b.bits_per_b(),+ b_group_size: arguments.b.group_size(),+ signed_codes: arguments.b.signed_codes(),+ a_full_precision: matches!(arguments.a, MatmulA::FullPrecision { .. }),+ gathered: arguments.gather_indices.is_some(),+ d_transform: arguments.d_transform.mask(),+ }+ }++ pub fn is_quant(&self) -> bool {+ self.b_prologue != GemmBPrologueKind::FullPrecision+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum MatmulPath {+ Gemv,+ Gemm,+}diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rsindex b96024331..25adfc458 100644--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs@@ -1,11 +1,14 @@use crate::backends::common::Backend;+pub mod activation_transform;pub mod attention_gemm;pub mod delta_net_chunked_prefill;pub mod delta_net_tree_verify;pub mod matmul;pub mod radix_top_k_small;+pub use activation_transform::ActivationTransform;+include!(concat!(env!("OUT_DIR"), "/traits.rs"));pub trait Kernels: Sized {diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rsnew file mode 100644index 000000000..160660fe8--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/activation_transform.rs@@ -0,0 +1,96 @@+use half::bf16;+use num_traits::{Float, NumCast};+use proc_macros::kernel;++use super::{hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8};+use crate::{+ array::ArrayElement,+ backends::common::gpu_types::{ActivationTransformOp, HADAMARD_TRANSFORM_BLOCK_SIZE},+};++#[kernel(ActivationTransform)]+#[variants(T, f32, bf16)]+pub fn activation_transform<T: ArrayElement + Float>(+ #[optional(!in_place)] input: Option<*const T>,+ #[optional(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)] fp_out: Option<+ *mut T,+ >,+ #[optional(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)]+ q_out: Option<*mut i8>,+ #[optional(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)]+ scales_out: Option<*mut f32>,+ #[optional(ops == ActivationTransformOp::QuantizeWithGroupSums)] group_sums_out: Option<*mut i32>,+ rht_factors: *const i32,+ batch_size: u32,+ element_count: u32,+ #[specialize] ops: ActivationTransformOp,+ #[specialize] in_place: bool,+) {+ let input = match in_place {+ true => fp_out.expect("in-place transform requires fp_out"),+ false => input.expect("out-of-place transform requires input"),+ };+ let rows = batch_size as usize;+ let columns = element_count as usize;+ let input_rht = ops != ActivationTransformOp::OutputRht;+ let quantize = matches!(ops, ActivationTransformOp::Quantize | ActivationTransformOp::QuantizeWithGroupSums);++ let groups = columns / HADAMARD_TRANSFORM_BLOCK_SIZE;+ let mut transformed = vec![0.0f32; columns];+ for row in 0..rows {+ let row_offset = row * columns;+ for stripe_start in (0..columns).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE) {+ let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE];+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {+ let index = stripe_start + lane;+ let value: f32 = NumCast::from(unsafe { *input.add(row_offset + index) }).unwrap();+ let factor = unsafe { *rht_factors.add(index) } as f32;+ stripe[lane] = if input_rht {+ value * factor+ } else {+ value+ };+ }++ hadamard_transform(&mut stripe);++ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {+ let index = stripe_start + lane;+ let factor = unsafe { *rht_factors.add(index) } as f32;+ transformed[index] = if input_rht {+ stripe[lane]+ } else {+ stripe[lane] * factor+ };+ }+ }++ if quantize {+ let q_out = q_out.expect("quantized transform requires q_out");+ let scales_out = scales_out.expect("quantized transform requires scales_out");+ for group in 0..groups {+ let start = group * HADAMARD_TRANSFORM_BLOCK_SIZE;+ let end = start + HADAMARD_TRANSFORM_BLOCK_SIZE;+ let slice = &transformed[start..end];+ let divisor = min_max_symmetric_divisor(slice);+ unsafe { *scales_out.add(row * groups + group) = divisor };+ let mut group_sum = 0i32;+ for index in start..end {+ let q = quantize_symmetric_i8(transformed[index], divisor);+ unsafe { *q_out.add(row * columns + index) = q };+ group_sum += q as i32;+ }+ if let Some(group_sums_out) = group_sums_out {+ unsafe { *group_sums_out.add(row * groups + group) = group_sum };+ }+ }+ } else {+ let fp_out = fp_out.expect("FP transform requires fp_out");+ for index in 0..columns {+ unsafe {+ *fp_out.add(row_offset + index) = <T as NumCast>::from(transformed[index]).unwrap();+ }+ }+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rsnew file mode 100644index 000000000..3c3defd42--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/activation_transform/mod.rs@@ -0,0 +1,42 @@+pub mod activation_transform;++use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;++pub const INT8_SYMMETRIC_QUANTIZATION_MAXIMUM: f32 = 127.0;++pub fn min_max_symmetric_divisor(values: &[f32]) -> f32 {+ let (min, max) =+ values.iter().fold((f32::INFINITY, f32::NEG_INFINITY), |(min, max), &value| (min.min(value), max.max(value)));+ let magnitude = min.abs().max(max.abs());+ if magnitude.is_finite() && magnitude > 0.0 {+ magnitude / INT8_SYMMETRIC_QUANTIZATION_MAXIMUM+ } else {+ 1.0+ }+}++pub fn quantize_symmetric_i8(+ value: f32,+ divisor: f32,+) -> i8 {+ (value / divisor).round().clamp(-INT8_SYMMETRIC_QUANTIZATION_MAXIMUM, INT8_SYMMETRIC_QUANTIZATION_MAXIMUM) as i8+}++pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE]) {+ let mut stride = 1;+ while stride < HADAMARD_TRANSFORM_BLOCK_SIZE {+ for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {+ if lane & stride == 0 {+ let a = values[lane];+ let b = values[lane | stride];+ values[lane] = a + b;+ values[lane | stride] = a - b;+ }+ }+ stride <<= 1;+ }+ let scale = 1.0 / (HADAMARD_TRANSFORM_BLOCK_SIZE as f32).sqrt();+ for v in values.iter_mut() {+ *v *= scale;+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rsdeleted file mode 100644index d43c3cdba..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs+++ /dev/null@@ -1,70 +0,0 @@-use half::bf16;-use num_traits::{Float, NumCast};-use proc_macros::kernel;--use crate::{- array::ArrayElement,- backends::common::gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},-};--pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE]) {- let mut stride = 1;- while stride < HADAMARD_TRANSFORM_BLOCK_SIZE {- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {- if lane & stride == 0 {- let a = values[lane];- let b = values[lane | stride];- values[lane] = a + b;- values[lane | stride] = a - b;- }- }- stride <<= 1;- }- let scale = 1.0 / (HADAMARD_TRANSFORM_BLOCK_SIZE as f32).sqrt();- for v in values.iter_mut() {- *v *= scale;- }-}--#[kernel(HadamardTransform)]-#[variants(T, f32, bf16)]-pub fn hadamard_transform_mul<T: ArrayElement + Float>(- data: *mut T,- factors: *const i32,- hidden_dim: u32,- batch_size: u32,- #[specialize] transform_order: HadamardTransformOrder,-) {- let hidden_dim = hidden_dim as usize;- let batch_size = batch_size as usize;- assert!(- hidden_dim.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE),- "hidden_dim must be a multiple of {HADAMARD_TRANSFORM_BLOCK_SIZE}"- );-- for batch in 0..batch_size {- let row_offset = batch * hidden_dim;- for stripe_start in (0..hidden_dim).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE) {- let mut stripe = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE];- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {- let v: f32 = NumCast::from(unsafe { *data.add(row_offset + stripe_start + lane) }).unwrap();- let f = unsafe { *factors.add(stripe_start + lane) } as f32;- stripe[lane] = match transform_order {- HadamardTransformOrder::Input => v * f,- HadamardTransformOrder::Output => v,- };- }-- hadamard_transform(&mut stripe);-- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {- let f = unsafe { *factors.add(stripe_start + lane) } as f32;- let result = match transform_order {- HadamardTransformOrder::Input => stripe[lane],- HadamardTransformOrder::Output => stripe[lane] * f,- };- unsafe { *data.add(row_offset + stripe_start + lane) = <T as NumCast>::from(result).unwrap() };- }- }- }-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/mod.rsdeleted file mode 100644index 1259c9457..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod hadamard_transform;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rsindex 486d537d1..e9b8b936e 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs@@ -3,9 +3,9 @@ use crate::{backends::{common::{Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, BufferArg, Encoder, Kernels,- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder, QuantizationMode},+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMode},kernel::{- HadamardTransformKernel, TensorAddBiasKernel,+ ActivationTransform, TensorAddBiasKernel,matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulKernel},},},@@ -19,7 +19,7 @@ pub struct MatmulCpuKernel {weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,+ output_rht: ActivationTransform<Cpu>,bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,}@@ -37,11 +37,7 @@ impl MatmulKernel for MatmulCpuKernel {return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());}}- let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(- context,- output_data_type,- HadamardTransformOrder::Output,- )?;+ let output_rht = ActivationTransform::output_rht(context, output_data_type, true)?;let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context,output_data_type,@@ -53,7 +49,7 @@ impl MatmulKernel for MatmulCpuKernel {weights_data_type,input_data_type,output_data_type,- hadamard,+ output_rht,bias_add,})}@@ -232,21 +228,20 @@ impl MatmulKernel for MatmulCpuKernel {biases,bits,group_size,+ signed_codes,} => {let (num_groups_k, zero_point_stride, pack_factor) = quant_layout.unwrap();let weight_linear_index = b_col * k_u + inner;- let quantized_value = if *bits == 4 {- let word_index = weight_linear_index / pack_factor;- let bit_offset = (weight_linear_index % pack_factor) * 4;- let w = weights.as_ptr() as *const u32;- let nibble = ((w.add(word_index).read_unaligned() >> bit_offset) & 0xF) as u8;- f32::from(nibble)- } else {- let word_index = weight_linear_index / pack_factor;- let bit_offset = (weight_linear_index % pack_factor) * 8;- let w = weights.as_ptr() as *const u32;- ((w.add(word_index).read_unaligned() >> bit_offset) & 0xFF) as f32- };+ let word_index = weight_linear_index / pack_factor;+ let bit_offset = (weight_linear_index % pack_factor) * *bits;+ let weights_words = weights.as_ptr() as *const u32;+ let word = weights_words.add(word_index).read_unaligned();+ let code_mask = (1u32 << bits) - 1;+ let mut weight_code = ((word >> bit_offset) & code_mask) as u8;+ if *signed_codes {+ weight_code ^= 1u8 << (bits - 1);+ }+ let quantized_value = f32::from(weight_code);let group_index = inner / group_size;let scale = read_f32(scales.as_ptr(),@@ -298,7 +293,7 @@ impl MatmulKernel for MatmulCpuKernel {});if let Some(factors) = post_rht {- self.hadamard.encode(&mut *d, factors, n, m, encoder);+ self.output_rht.encode_fp_in_place(&mut *d, factors, m, n, encoder);if let Some(bias) = bias_alloc {let output_length = m.checked_mul(n).expect("matmul output length must fit in u32");self.bias_add.encode(diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rsindex 686ce50db..41c92a735 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs@@ -22,6 +22,7 @@ pub(super) enum WeightData {biases: Option<SendPtr<u8>>,bits: usize,group_size: usize,+ signed_codes: bool,},}@@ -63,6 +64,7 @@ impl WeightData {biases,mode,group_size,+ signed_codes,} => WeightData::Quantized {weights: alloc_ptr(weights),scales: alloc_ptr(scales),@@ -70,6 +72,7 @@ impl WeightData {biases: Some(alloc_ptr(biases)),bits: bits_of(mode),group_size: group_size as usize,+ signed_codes,},MatmulB::ScaleZeroPointDequant {b: weights,@@ -77,6 +80,7 @@ impl WeightData {zero_points,mode,group_size,+ signed_codes,} => WeightData::Quantized {weights: alloc_ptr(weights),scales: alloc_ptr(scales),@@ -84,12 +88,14 @@ impl WeightData {biases: None,bits: bits_of(mode),group_size: group_size as usize,+ signed_codes,},MatmulB::ScaleSymmetricDequant {b: weights,scales,mode,group_size,+ signed_codes,} => WeightData::Quantized {weights: alloc_ptr(weights),scales: alloc_ptr(scales),@@ -97,6 +103,7 @@ impl WeightData {biases: None,bits: bits_of(mode),group_size: group_size as usize,+ signed_codes,},}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rsindex 9edb8a195..9541bd94c 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs@@ -3,18 +3,17 @@ use std::convert::Infallible;use crate::backends::{common::Kernels, cpu::Cpu};mod activation;+pub(crate) mod activation_transform;mod attention;mod embedding;mod gated_act_mul;mod gdn;-mod hadamard_transform;mod logit_soft_cap;mod matmul;mod moe;mod normalization;mod pooling;mod radix_top_k_small;-pub(crate) mod rht_quantize_activations;mod sampling;mod short_conv;mod softmax;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/mod.rsdeleted file mode 100644index d463aeba0..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/mod.rs+++ /dev/null@@ -1,21 +0,0 @@-pub mod rht_quantize_activations;--pub const INT8_SYMMETRIC_QUANTIZATION_MAXIMUM: f32 = 127.0;--pub fn min_max_symmetric_divisor(values: &[f32]) -> f32 {- let (min, max) =- values.iter().fold((f32::INFINITY, f32::NEG_INFINITY), |(min, max), &value| (min.min(value), max.max(value)));- let magnitude = min.abs().max(max.abs());- if magnitude.is_finite() && magnitude > 0.0 {- magnitude / INT8_SYMMETRIC_QUANTIZATION_MAXIMUM- } else {- 1.0- }-}--pub fn quantize_symmetric_i8(- value: f32,- divisor: f32,-) -> i8 {- (value / divisor).round().clamp(-INT8_SYMMETRIC_QUANTIZATION_MAXIMUM, INT8_SYMMETRIC_QUANTIZATION_MAXIMUM) as i8-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs b/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rsdeleted file mode 100644index 2899848af..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/rht_quantize_activations/rht_quantize_activations.rs+++ /dev/null@@ -1,63 +0,0 @@-use half::bf16;-use num_traits::{Float, NumCast};-use proc_macros::kernel;--use super::{- super::hadamard_transform::hadamard_transform::hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8,-};-use crate::{array::ArrayElement, backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE};--#[kernel(RHTQuantizeActivations)]-#[variants(InputT, f32, bf16)]-pub fn activations_prepare<InputT: ArrayElement + Float>(- input: *const InputT,- q_out: *mut i8,- scales_out: *mut f32,- #[optional(emit_group_sums)] group_sums_out: Option<*mut i32>,- rht_factors: *const i32,- batch_size: u32,- element_count: u32,- group_size: u32,- #[allow(unused)]- #[specialize]- emit_group_sums: bool,-) {- let rows = batch_size as usize;- let columns = element_count as usize;- let group_size = group_size as usize;- assert!(columns.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE));- assert!(group_size > 0 && columns.is_multiple_of(group_size));-- let groups = columns.div_ceil(group_size);- let mut prepared = vec![0.0f32; columns];- for row in 0..rows {- for block_start in (0..columns).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE) {- let mut block = [0.0f32; HADAMARD_TRANSFORM_BLOCK_SIZE];- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE {- let index = block_start + lane;- let value: f32 = NumCast::from(unsafe { *input.add(row * columns + index) }).unwrap();- let factor = unsafe { *rht_factors.add(index) } as f32;- block[lane] = value * factor;- }- hadamard_transform(&mut block);- prepared[block_start..block_start + HADAMARD_TRANSFORM_BLOCK_SIZE].copy_from_slice(&block);- }-- for group in 0..groups {- let start = group * group_size;- let end = (start + group_size).min(columns);- let slice = &prepared[start..end];- let divisor = min_max_symmetric_divisor(slice);- unsafe { *scales_out.add(row * groups + group) = divisor };- let mut group_sum = 0i32;- for index in start..end {- let q = quantize_symmetric_i8(prepared[index], divisor);- unsafe { *q_out.add(row * columns + index) = q };- group_sum += q as i32;- }- if let Some(group_sums_out) = group_sums_out {- unsafe { *group_sums_out.add(row * groups + group) = group_sum };- }- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metalnew file mode 100644index 000000000..e2081c52d--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/activation_transform/activation_transform.metal@@ -0,0 +1,66 @@+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../generated/activation_transform.h"+#include "../hadamard_transform/hadamard_transform.h"++using namespace metal;+using namespace uzu::activation_transform;++UZU_CONST float SYM_QMAX = 127.0;++template <typename T>+VARIANTS(T, float, bfloat)+PUBLIC KERNEL(ActivationTransform)(+ const device T* input OPTIONAL(!in_place),+ device T* fp_out OPTIONAL(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht),+ device int8_t* q_out OPTIONAL(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums),+ device float* scales_out OPTIONAL(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums),+ device int32_t* group_sums_out OPTIONAL(ops == ActivationTransformOp::QuantizeWithGroupSums),+ const device int32_t* rht_factors,+ constant uint& batch_size,+ constant uint& element_count,+ const ActivationTransformOp ops SPECIALIZE,+ const bool in_place SPECIALIZE,+ uint block_index GROUPS(element_count.div_ceil(METAL_SIMD_SIZE)),+ uint batch_index GROUPS(batch_size),+ uint lane_index THREADS(METAL_SIMD_SIZE)+) {+ if (in_place) {+ input = reinterpret_cast<const device T*>(fp_out);+ }++ const bool input_rht = ops != ActivationTransformOp::OutputRht;+ const bool quantize = ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums;+ const uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;+ const uint element_index = batch_index * element_count + factor_index;++ float value = static_cast<float>(input[element_index]);+ if (input_rht) {+ value = simdgroup_input_random_hadamard_transform(lane_index, value, rht_factors[factor_index]);+ } else {+ value = simdgroup_output_random_hadamard_transform(lane_index, value, rht_factors[factor_index]);+ }++ if (quantize) {+ const float magnitude = max(fabs(simd_min(value)), fabs(simd_max(value)));+ const float scale = isfinite(magnitude) && magnitude > 0.0f ? magnitude / SYM_QMAX : 1.0f;++ const int8_t code = static_cast<int8_t>(clamp(round(value / scale), -SYM_QMAX, SYM_QMAX));+ q_out[element_index] = code;++ const uint group_index = batch_index * (element_count / METAL_SIMD_SIZE) + block_index;+ if (lane_index == 0) {+ scales_out[group_index] = scale;+ }++ if (ops == ActivationTransformOp::QuantizeWithGroupSums) {+ const int group_sum = simd_sum(int(code));+ if (lane_index == 0) {+ group_sums_out[group_index] = group_sum;+ }+ }+ } else {+ fp_out[element_index] = static_cast<T>(value);+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 77ce9233a..728c090e4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -3,7 +3,7 @@#include "../common/thread_context.h"#include "../matmul/common/fragment.h"#include "../matmul/common/loader.h"-#include "../matmul/common/mxu_fragment_ops.h"+#include "../matmul/common/mxu_fragment/ops.h"#include "../matmul/common/simdgroup_fragment_ops.h"#include "../generated/ring.h"#include "../generated/trie.h"diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metalindex 652a9c8e9..435995901 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/chunked/output_and_state.metal@@ -3,7 +3,7 @@#include "../../common/dsl.h"#include "../../common/thread_context.h"#include "../../matmul/common/fragment.h"-#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/mxu_fragment/ops.h"#include "../../matmul/common/simdgroup_fragment_ops.h"using namespace metal;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.hindex e374a97bb..e4eb38a83 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/common/gram.h@@ -2,7 +2,7 @@#include "../../common/defines.h"#include "../../matmul/common/fragment.h"-#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/mxu_fragment/ops.h"#include "../../matmul/common/simdgroup_fragment_ops.h"#include <metal_stdlib>diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metalindex 310aa8050..50022cee9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/out.metal@@ -3,7 +3,7 @@#include "../../common/dsl.h"#include "../../common/thread_context.h"#include "../../matmul/common/fragment.h"-#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/mxu_fragment/ops.h"#include "../../matmul/common/simdgroup_fragment_ops.h"using namespace metal;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metalindex 75a681e98..54c28d4a4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/tree_gram.metal@@ -4,7 +4,7 @@#include "../../common/thread_context.h"#include "../../generated/trie.h"#include "../../matmul/common/fragment.h"-#include "../../matmul/common/mxu_fragment_ops.h"+#include "../../matmul/common/mxu_fragment/ops.h"#include "../../matmul/common/simdgroup_fragment_ops.h"#include "../common/gram.h"#include "../common/tri_inv.h"diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/activation_transform.h b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_transform.hnew file mode 100644index 000000000..70797b2f0--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/activation_transform.h@@ -0,0 +1,14 @@+// Auto-generated from gpu_types/activation_transform - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::activation_transform {+enum class ActivationTransformOp : uint32_t {+ InputRht = 0,+ OutputRht = 1,+ Quantize = 2,+ QuantizeWithGroupSums = 3,+};+} // namespace uzu::activation_transformdiff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.hindex d94188451..d36ab48c9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/hadamard_order.h@@ -6,9 +6,4 @@ using namespace metal;namespace uzu::hadamard_order {static constant constexpr size_t HADAMARD_TRANSFORM_BLOCK_SIZE = 32;--enum class HadamardTransformOrder : uint32_t {- Input = 0,- Output = 1,-};} // namespace uzu::hadamard_orderdiff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metaldeleted file mode 100644index 992a04095..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal+++ /dev/null@@ -1,32 +0,0 @@-#include <metal_stdlib>-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../generated/hadamard_order.h"-#include "hadamard_transform.h"--using namespace metal;-using namespace uzu::hadamard_order;--template <typename T>-VARIANTS(T, bfloat, float)-PUBLIC KERNEL(HadamardTransform)(- device T* data,- const device int32_t* factors,- constant uint& hidden_dim,- constant uint& batch_size,- const HadamardTransformOrder transform_order SPECIALIZE,- uint block_index GROUPS(hidden_dim.div_ceil(METAL_SIMD_SIZE)),- uint batch_index GROUPS(batch_size),- uint lane_index THREADS(METAL_SIMD_SIZE)-) {- uint factor_index = block_index * HADAMARD_TRANSFORM_BLOCK_SIZE + lane_index;- uint element_index = batch_index * hidden_dim + factor_index;-- if (transform_order == HadamardTransformOrder::Input) {- data[element_index] =- simdgroup_input_random_hadamard_transform(lane_index, data[element_index], factors[factor_index]);- } else {- data[element_index] =- simdgroup_output_random_hadamard_transform(lane_index, data[element_index], factors[factor_index]);- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/cooperative_vectors.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/cooperative_vectors.hnew file mode 100644index 000000000..e73cb1c5c--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/cooperative_vectors.h@@ -0,0 +1,45 @@+template <typename CooperativeTensor, typename U>+METAL_FUNC static void load_paired_vectors(+ thread CooperativeTensor& cooperative,+ const thread ThreadVector<U>& vector_0,+ const thread ThreadVector<U>& vector_1+) {+ if constexpr (RELAXED) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative[i] = vector_0[i];+ cooperative[ELEMENTS_PER_THREAD + i] = vector_1[i];+ }+ } else {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < 4; i++) {+ cooperative[i] = vector_0[i];+ cooperative[4 + i] = vector_1[i];+ cooperative[8 + i] = vector_0[4 + i];+ cooperative[12 + i] = vector_1[4 + i];+ }+ }+}++template <typename CooperativeTensor, typename U>+METAL_FUNC static void store_paired_vectors(+ thread CooperativeTensor& cooperative,+ thread ThreadVector<U>& vector_0,+ thread ThreadVector<U>& vector_1+) {+ if constexpr (RELAXED) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ vector_0[i] = cooperative[i];+ vector_1[i] = cooperative[ELEMENTS_PER_THREAD + i];+ }+ } else {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < 4; i++) {+ vector_0[i] = cooperative[i];+ vector_1[i] = cooperative[4 + i];+ vector_0[4 + i] = cooperative[8 + i];+ vector_1[4 + i] = cooperative[12 + i];+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.hnew file mode 100644index 000000000..643b9a7b6--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.h@@ -0,0 +1,53 @@+template <typename RightElement, class OutputFragment, class LeftFragment>+METAL_FUNC static void fragment_matmul_int8_device_weights(+ thread OutputFragment& output,+ thread LeftFragment& left,+ const device uchar* right_signed_codes,+ const int right_row_stride_bytes+) {+ static_assert(RELAXED, "device weight tensors require the relaxed MXU layout");+ static_assert(LeftFragment::COL_FRAGMENTS == 2, "device weight tensors expect K tiled as two fragments");+ static_assert(OutputFragment::COL_FRAGMENTS % 2 == 0, "device weight tensors require even N fragments");+ static_assert(LeftFragment::ROW_FRAGMENTS == OutputFragment::ROW_FRAGMENTS, "M tiles must match");++ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;+ constexpr int tile_k = int(2 * FRAGMENT_COLS);+ constexpr int tile_n = int(2 * FRAGMENT_COLS);+ constexpr int element_bits = metal::is_same_v<RightElement, metal::int4b_format> ? 4 : 8;+ constexpr int elements_per_byte = 8 / element_bits;+ using RightTensor = tensor<device RightElement, extents<int, tile_k, tile_n>, tensor_inline>;+ using RightPointer =+ metal::conditional_t<metal::is_same_v<RightElement, metal::int4b_format>, device uchar*, device RightElement*>;++ constexpr auto descriptor =+ mpp::tensor_ops::matmul2d_descriptor(FRAGMENT_ROWS, tile_n, tile_k, false, true, RELAXED, MatmulMode::multiply);+ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;++ const array<int, 2> right_strides = {1, right_row_stride_bytes * elements_per_byte};++ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < rows; ++row) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < cols; col += 2) {+ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<int8_t, RightElement, int>();+ load_paired_vectors(cooperative_left, left.fragment_at(row, 0), left.fragment_at(row, 1));++ RightTensor right_tensor(+ // MPP rejects const-qualified tensor element types.+ reinterpret_cast<RightPointer>(+ const_cast<device uchar*>(right_signed_codes + int(col * FRAGMENT_COLS) * right_row_stride_bytes)+ ),+ extents<int, tile_k, tile_n>{},+ right_strides+ );++ auto cooperative_output =+ matmul_op.template get_destination_cooperative_tensor<decltype(cooperative_left), RightTensor, int>();++ matmul_op.run(cooperative_left, right_tensor, cooperative_output);++ store_paired_vectors(cooperative_output, output.fragment_at(row, col), output.fragment_at(row, col + 1));+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.hnew file mode 100644index 000000000..b0e441729--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h@@ -0,0 +1,107 @@+template <+ bool ACCUMULATE,+ bool transpose_a,+ bool transpose_b,+ class OutputFragment,+ class LeftFragment,+ class RightFragment>+METAL_FUNC static void fragment_matmul(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+) {+ constexpr ushort left_rows = transpose_a ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;+ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;+ static_assert(left_rows == rows, "fragment matmul: M dimensions do not match");++ constexpr ushort right_cols = transpose_b ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;+ static_assert(right_cols == cols, "fragment matmul: N dimensions do not match");++ constexpr ushort left_depth = transpose_a ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;+ constexpr ushort depth = transpose_b ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;+ static_assert(left_depth == depth, "fragment matmul: K dimensions do not match");++ static_assert(+ (cols % 2 == 0) || (cols == 1 && rows % 2 == 0),+ "MXU fragment_mma requires even N, or N==1 with even M (MPP pairing)"+ );++ constexpr auto transpose_left = metal::bool_constant<transpose_a>{};+ constexpr auto transpose_right = metal::bool_constant<transpose_b>{};+ constexpr bool pair_output_rows = (cols == 1 && rows % 2 == 0);++ auto matmul_paired_outputs = [&](ushort row, ushort col, ushort depth_index, auto use_multiply_accumulate) {+ constexpr bool matmul_accumulate = decltype(use_multiply_accumulate)::value;+ if constexpr (pair_output_rows) {+ matmul<+ matmul_accumulate,+ typename OutputFragment::ElementType,+ typename LeftFragment::ElementType,+ typename RightFragment::ElementType,+ transpose_a,+ transpose_b>(+ output.fragment_at(row, col),+ output.fragment_at(row + 1, col),+ left.fragment_at(row, depth_index, transpose_left),+ left.fragment_at(row + 1, depth_index, transpose_left),+ transpose_left,+ right.fragment_at(depth_index, col, transpose_right),+ transpose_right+ );+ } else {+ matmul<+ matmul_accumulate,+ typename OutputFragment::ElementType,+ typename LeftFragment::ElementType,+ typename RightFragment::ElementType,+ transpose_a,+ transpose_b>(+ output.fragment_at(row, col),+ output.fragment_at(row, col + 1),+ left.fragment_at(row, depth_index, transpose_left),+ transpose_left,+ right.fragment_at(depth_index, col, transpose_right),+ right.fragment_at(depth_index, col + 1, transpose_right),+ transpose_right+ );+ }+ };++ constexpr ushort output_row_step = pair_output_rows ? 2 : 1;+ constexpr ushort output_col_count = pair_output_rows ? 1 : cols;+ constexpr ushort output_col_step = pair_output_rows ? 1 : 2;++ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < rows; row += output_row_step) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < output_col_count; col += output_col_step) {+ if constexpr (!ACCUMULATE) {+ matmul_paired_outputs(row, col, 0, metal::bool_constant<false>{});+ }+ METAL_PRAGMA_UNROLL+ for (ushort depth_index = ACCUMULATE ? 0 : 1; depth_index < depth; ++depth_index) {+ matmul_paired_outputs(row, col, depth_index, metal::bool_constant<true>{});+ }+ }+ }+}++template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+METAL_FUNC static void fragment_mma(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+) {+ fragment_matmul<true, transpose_a, transpose_b>(output, left, right);+}++template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+METAL_FUNC static void fragment_mm(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+) {+ // MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.+ fragment_matmul<false, transpose_a, transpose_b>(output, left, right);+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/layout.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/layout.hnew file mode 100644index 000000000..e2b2d5639--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/layout.h@@ -0,0 +1,45 @@+UZU_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;+UZU_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;+UZU_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;+using BlockStorage = DeviceBlockStorage;++UZU_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;++UZU_CONST ushort THREAD_ELEMENT_ROWS = 2;+UZU_CONST ushort THREAD_ELEMENT_COLS = 4;++UZU_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;++static_assert(+ THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,+ "MxuFragment shape is not consistent with element count"+);++template <typename U>+using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;++METAL_FUNC static constexpr short2 get_position(ushort simd_lane_id) {+ if constexpr (RELAXED) {+ const short quad = simd_lane_id / 4;+ const short row = (quad & 4) + (simd_lane_id / 2) % 4;+ const short col = ((quad & 2) + simd_lane_id % 2) * THREAD_ELEMENT_COLS;+ return short2{col, row};+ } else {+ const short col = short((simd_lane_id & 1) * 2 + ((simd_lane_id >> 3) & 1) * 4);+ const short row = short(((simd_lane_id >> 1) & 3) + ((simd_lane_id >> 4) & 1) * 4);+ return short2{col, row};+ }+}++METAL_FUNC static constexpr short2 get_element_offset(ushort element_index) {+ if constexpr (RELAXED) {+ const short row = short((element_index / THREAD_ELEMENT_COLS) * THREAD_ELEMENT_ROW_STRIDE);+ const short col = short(element_index % THREAD_ELEMENT_COLS);+ return short2{col, row};+ } else {+ const short row = short((element_index / 4) * 8);+ const ushort col_slot = element_index & 3;+ const short col = short((col_slot & 1) + (col_slot / 2) * 8);+ return short2{col, row};+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.hnew file mode 100644index 000000000..88619860a--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h@@ -0,0 +1,36 @@+#pragma once++#include <metal_stdlib>++#include "../../../common/integral_constant.h"+#include "../../../common/thread_context.h"++#include "../defines.h"+#include "../loader.h"++#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>++using namespace metal;++namespace uzu {+namespace matmul {++UZU_CONST ushort MXU_MMA_ROWS = 16;+UZU_CONST ushort MXU_MMA_COLS = 16;++// RELAXED=false uses the strict MPP layout; it currently performs about the same as simdgroup.+template <bool RELAXED = true>+struct MxuFragmentOps {+ using MatmulMode = mpp::tensor_ops::matmul2d_descriptor::mode;++#include "layout.h"+#include "cooperative_vectors.h"+#include "tile_matmul.h"+#include "fragment_matmul.h"+#include "device_weight_matmul.h"+};++using MxuStrictFragmentOps = MxuFragmentOps<false>;++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.hnew file mode 100644index 000000000..e54ad95aa--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.h@@ -0,0 +1,102 @@+// MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.+template <+ bool ACCUMULATE,+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a,+ bool transpose_b,+ typename MarshalInputs>+METAL_FUNC static void mma_impl(+ thread ThreadVector<CType>& output_0,+ thread ThreadVector<CType>& output_1,+ MarshalInputs marshal_inputs+) {+ constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(+ FRAGMENT_ROWS,+ 2 * FRAGMENT_COLS,+ FRAGMENT_COLS,+ transpose_a,+ transpose_b,+ RELAXED,+ ACCUMULATE ? MatmulMode::multiply_accumulate : MatmulMode::multiply+ );++ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;++ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<AType, BType, CType>();+ auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<AType, BType, CType>();+ auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<+ decltype(cooperative_left),+ decltype(cooperative_right),+ CType>();++ marshal_inputs(cooperative_left, cooperative_right);++ if constexpr (ACCUMULATE) {+ load_paired_vectors(cooperative_output, output_0, output_1);+ }++ matmul_op.run(cooperative_left, cooperative_right, cooperative_output);++ store_paired_vectors(cooperative_output, output_0, output_1);+}++template <+ bool ACCUMULATE,+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a = false,+ bool transpose_b = false>+METAL_FUNC static void matmul(+ thread ThreadVector<CType>& output_col_0,+ thread ThreadVector<CType>& output_col_1,+ const thread ThreadVector<AType>& left,+ metal::bool_constant<transpose_a>,+ const thread ThreadVector<BType>& right_col_0,+ const thread ThreadVector<BType>& right_col_1,+ metal::bool_constant<transpose_b>+) {+ mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(+ output_col_0,+ output_col_1,+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_left[i] = left[i];+ }+ load_paired_vectors(cooperative_right, right_col_0, right_col_1);+ }+ );+}++template <+ bool ACCUMULATE,+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a = false,+ bool transpose_b = false>+METAL_FUNC static void matmul(+ thread ThreadVector<CType>& output_row_0,+ thread ThreadVector<CType>& output_row_1,+ const thread ThreadVector<AType>& left_row_0,+ const thread ThreadVector<AType>& left_row_1,+ metal::bool_constant<transpose_a>,+ const thread ThreadVector<BType>& right,+ metal::bool_constant<transpose_b>+) {+ static_assert(RELAXED, "strict MXU row-pairing is not implemented");+ mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(+ output_row_0,+ output_row_1,+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {+ load_paired_vectors(cooperative_left, left_row_0, left_row_1);+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_right[i] = right[i];+ }+ }+ );+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.hdeleted file mode 100644index d4e77f502..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h+++ /dev/null@@ -1,365 +0,0 @@-#pragma once--#include <metal_simdgroup>-#include <metal_simdgroup_matrix>-#include <metal_stdlib>--#include "../../common/integral_constant.h"-#include "../../common/thread_context.h"-using namespace uzu;--#include "defines.h"-#include "loader.h"--#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>--using namespace metal;--namespace uzu {-namespace matmul {--UZU_CONST ushort MXU_MMA_ROWS = 16;-UZU_CONST ushort MXU_MMA_COLS = 16;--// RELAXED=false uses the strict MPP layout; it currently performs about the same as simdgroup.-template <bool RELAXED = true>-struct MxuFragmentOps {- UZU_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;- UZU_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;- UZU_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;- using BlockStorage = DeviceBlockStorage;-- UZU_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;-- UZU_CONST ushort THREAD_ELEMENT_ROWS = 2;- UZU_CONST ushort THREAD_ELEMENT_COLS = 4;-- UZU_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;-- static_assert(- THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,- "MxuFragment shape is not consistent with element count"- );-- template <typename U>- using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;-- METAL_FUNC static constexpr short2 get_position(ushort simd_lane_id) {- if constexpr (RELAXED) {- const short quad = simd_lane_id / 4;- const short row = (quad & 4) + (simd_lane_id / 2) % 4;- const short col = ((quad & 2) + simd_lane_id % 2) * THREAD_ELEMENT_COLS;- return short2{col, row};- } else {- const short col = short((simd_lane_id & 1) * 2 + ((simd_lane_id >> 3) & 1) * 4);- const short row = short(((simd_lane_id >> 1) & 3) + ((simd_lane_id >> 4) & 1) * 4);- return short2{col, row};- }- }-- METAL_FUNC static constexpr short2 get_element_offset(ushort element_index) {- if constexpr (RELAXED) {- const short row = short((element_index / THREAD_ELEMENT_COLS) * THREAD_ELEMENT_ROW_STRIDE);- const short col = short(element_index % THREAD_ELEMENT_COLS);- return short2{col, row};- } else {- const short row = short((element_index / 4) * 8);- const ushort col_slot = element_index & 3;- const short col = short((col_slot & 1) + (col_slot / 2) * 8);- return short2{col, row};- }- }-- template <typename CooperativeTensor, typename U>- METAL_FUNC static void load_paired_vectors(- thread CooperativeTensor& cooperative,- const thread ThreadVector<U>& vector_0,- const thread ThreadVector<U>& vector_1- ) {- if constexpr (RELAXED) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- cooperative[i] = vector_0[i];- cooperative[ELEMENTS_PER_THREAD + i] = vector_1[i];- }- } else {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < 4; i++) {- cooperative[i] = vector_0[i];- cooperative[4 + i] = vector_1[i];- cooperative[8 + i] = vector_0[4 + i];- cooperative[12 + i] = vector_1[4 + i];- }- }- }-- template <typename CooperativeTensor, typename U>- METAL_FUNC static void store_paired_vectors(- thread CooperativeTensor& cooperative,- thread ThreadVector<U>& vector_0,- thread ThreadVector<U>& vector_1- ) {- if constexpr (RELAXED) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- vector_0[i] = cooperative[i];- vector_1[i] = cooperative[ELEMENTS_PER_THREAD + i];- }- } else {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < 4; i++) {- vector_0[i] = cooperative[i];- vector_1[i] = cooperative[4 + i];- vector_0[4 + i] = cooperative[8 + i];- vector_1[4 + i] = cooperative[12 + i];- }- }- }-- // MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.- template <- bool ACCUMULATE,- typename CType,- typename AType,- typename BType,- bool transpose_a,- bool transpose_b,- typename MarshalInputs>- METAL_FUNC static void mma_impl(- thread ThreadVector<CType>& output_0,- thread ThreadVector<CType>& output_1,- MarshalInputs marshal_inputs- ) {- constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(- FRAGMENT_ROWS,- 2 * FRAGMENT_COLS,- FRAGMENT_COLS,- transpose_a,- transpose_b,- RELAXED,- ACCUMULATE ? mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate- : mpp::tensor_ops::matmul2d_descriptor::mode::multiply- );-- mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;-- auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<AType, BType, CType>();- auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<AType, BType, CType>();- auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<- decltype(cooperative_left),- decltype(cooperative_right),- CType>();-- marshal_inputs(cooperative_left, cooperative_right);-- if constexpr (ACCUMULATE) {- load_paired_vectors(cooperative_output, output_0, output_1);- }-- matmul_op.run(cooperative_left, cooperative_right, cooperative_output);-- store_paired_vectors(cooperative_output, output_0, output_1);- }-- template <- bool ACCUMULATE,- typename CType,- typename AType,- typename BType,- bool transpose_a = false,- bool transpose_b = false>- METAL_FUNC static void matmul(- thread ThreadVector<CType>& output_col_0,- thread ThreadVector<CType>& output_col_1,- const thread ThreadVector<AType>& left,- metal::bool_constant<transpose_a>,- const thread ThreadVector<BType>& right_col_0,- const thread ThreadVector<BType>& right_col_1,- metal::bool_constant<transpose_b>- ) {- mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(- output_col_0,- output_col_1,- [&](thread auto& cooperative_left, thread auto& cooperative_right) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- cooperative_left[i] = left[i];- }- load_paired_vectors(cooperative_right, right_col_0, right_col_1);- }- );- }-- template <- bool ACCUMULATE,- typename CType,- typename AType,- typename BType,- bool transpose_a = false,- bool transpose_b = false>- METAL_FUNC static void matmul(- thread ThreadVector<CType>& output_row_0,- thread ThreadVector<CType>& output_row_1,- const thread ThreadVector<AType>& left_row_0,- const thread ThreadVector<AType>& left_row_1,- metal::bool_constant<transpose_a>,- const thread ThreadVector<BType>& right,- metal::bool_constant<transpose_b>- ) {- static_assert(RELAXED, "strict MXU row-pairing is not implemented");- mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(- output_row_0,- output_row_1,- [&](thread auto& cooperative_left, thread auto& cooperative_right) {- load_paired_vectors(cooperative_left, left_row_0, left_row_1);- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- cooperative_right[i] = right[i];- }- }- );- }-- template <- bool ACCUMULATE,- bool transpose_a,- bool transpose_b,- class OutputFragment,- class LeftFragment,- class RightFragment>- METAL_FUNC static void fragment_matmul(- thread OutputFragment& output,- thread LeftFragment& left,- thread RightFragment& right- ) {- constexpr ushort left_rows = transpose_a ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;- constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;- static_assert(left_rows == rows, "fragment matmul: M dimensions do not match");-- constexpr ushort right_cols = transpose_b ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;- constexpr ushort cols = OutputFragment::COL_FRAGMENTS;- static_assert(right_cols == cols, "fragment matmul: N dimensions do not match");-- constexpr ushort left_depth = transpose_a ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;- constexpr ushort depth = transpose_b ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;- static_assert(left_depth == depth, "fragment matmul: K dimensions do not match");-- static_assert(- (cols % 2 == 0) || (cols == 1 && rows % 2 == 0),- "MXU fragment_mma requires even N, or N==1 with even M (MPP pairing)"- );-- constexpr auto transpose_left = metal::bool_constant<transpose_a>{};- constexpr auto transpose_right = metal::bool_constant<transpose_b>{};-- if constexpr (cols == 1 && rows % 2 == 0) {- METAL_PRAGMA_UNROLL- for (ushort row = 0; row < rows; row += 2) {- METAL_PRAGMA_UNROLL- for (ushort col = 0; col < cols; ++col) {- if constexpr (!ACCUMULATE) {- matmul<- false,- typename OutputFragment::ElementType,- typename LeftFragment::ElementType,- typename RightFragment::ElementType,- transpose_a,- transpose_b>(- output.fragment_at(row, col),- output.fragment_at(row + 1, col),- left.fragment_at(row, 0, transpose_left),- left.fragment_at(row + 1, 0, transpose_left),- transpose_left,- right.fragment_at(0, col, transpose_right),- transpose_right- );- }- METAL_PRAGMA_UNROLL- for (ushort k = ACCUMULATE ? 0 : 1; k < depth; ++k) {- matmul<- true,- typename OutputFragment::ElementType,- typename LeftFragment::ElementType,- typename RightFragment::ElementType,- transpose_a,- transpose_b>(- output.fragment_at(row, col),- output.fragment_at(row + 1, col),- left.fragment_at(row, k, transpose_left),- left.fragment_at(row + 1, k, transpose_left),- transpose_left,- right.fragment_at(k, col, transpose_right),- transpose_right- );- }- }- }- } else if constexpr (cols % 2 == 0) {- METAL_PRAGMA_UNROLL- for (ushort row = 0; row < rows; ++row) {- METAL_PRAGMA_UNROLL- for (ushort col = 0; col < cols; col += 2) {- if constexpr (!ACCUMULATE) {- matmul<- false,- typename OutputFragment::ElementType,- typename LeftFragment::ElementType,- typename RightFragment::ElementType,- transpose_a,- transpose_b>(- output.fragment_at(row, col),- output.fragment_at(row, col + 1),- left.fragment_at(row, 0, transpose_left),- transpose_left,- right.fragment_at(0, col, transpose_right),- right.fragment_at(0, col + 1, transpose_right),- transpose_right- );- }- METAL_PRAGMA_UNROLL- for (ushort k = ACCUMULATE ? 0 : 1; k < depth; ++k) {- matmul<- true,- typename OutputFragment::ElementType,- typename LeftFragment::ElementType,- typename RightFragment::ElementType,- transpose_a,- transpose_b>(- output.fragment_at(row, col),- output.fragment_at(row, col + 1),- left.fragment_at(row, k, transpose_left),- transpose_left,- right.fragment_at(k, col, transpose_right),- right.fragment_at(k, col + 1, transpose_right),- transpose_right- );- }- }- }- }- }-- template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>- METAL_FUNC static void fragment_mma(- thread OutputFragment& output,- thread LeftFragment& left,- thread RightFragment& right- ) {- fragment_matmul<true, transpose_a, transpose_b>(output, left, right);- }-- template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>- METAL_FUNC static void fragment_mm(- thread OutputFragment& output,- thread LeftFragment& left,- thread RightFragment& right- ) {- // MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.- fragment_matmul<false, transpose_a, transpose_b>(output, left, right);- }-};--using MxuStrictFragmentOps = MxuFragmentOps<false>;--} // namespace matmul-} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.hindex 24fe2ba3b..b03a1b756 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h@@ -1,7 +1,7 @@#pragma once#include "fragment.h"-#include "mxu_fragment_ops.h"+#include "mxu_fragment/ops.h"#include "../../generated/matmul.h"using namespace metal;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.hindex f98167440..a72149d1e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h@@ -10,15 +10,16 @@ using namespace metal;namespace uzu {namespace gemm {-// Pre-scales each activation lane k by 2^-(BITS*k). qdot then reads the-// matching weight nibble/byte in place (value = q * 2^(BITS*k)) without-// shifting it down to the low bits; the positional 2^(BITS*k) factor cancels-// the 2^-(BITS*k) here, so the dot product is unchanged. All factors are powers-// of two, so this is bit-exact.+// For packed unsigned weights, pre-scales each activation lane k by+// 2^-(BITS*k). qdot then reads the matching nibble/byte in place (value =+// q * 2^(BITS*k)); the factors cancel, so the dot product is unchanged. All+// factors are powers of two, so this is bit-exact.template <typename T, typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U load_vector(const device T* x, thread U* x_thread) {+METAL_FUNC U load_vector(const device T* x, thread U* x_thread, const bool signed_codes) {using U4 = vec<U, 4>;- const U4 inv = U4(U(1), U(1) / U(1u << BITS), U(1) / U(1u << (2u * BITS)), U(1) / U(1u << (3u * BITS)));+ const U4 inv = (BITS == 8 && signed_codes)+ ? U4(U(1))+ : U4(U(1), U(1) / U(1u << BITS), U(1) / U(1u << (2u * BITS)), U(1) / U(1u << (3u * BITS)));U sum = 0;thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);METAL_PRAGMA_UNROLL@@ -46,7 +47,7 @@ METAL_FUNC U load_vector_safe(const device T* x, thread U* x_thread, int N) {}template <typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U qdot(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum) {+METAL_FUNC U qdot(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, const bool signed_codes) {static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");U accumulator = 0;@@ -54,34 +55,43 @@ METAL_FUNC U qdot(const device uint8_t* w, const thread U* x_thread, U scale, Uusing U4 = vec<U, 4>;const device ushort* weight_words = reinterpret_cast<const device ushort*>(w);const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);+ const ushort packed_mask = signed_codes ? 0x8888u : 0u;METAL_PRAGMA_UNROLLfor (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {// Mask each nibble in place (no shifts); value of lane k is n_k << (4*k),// i.e. n_k * 16^k, which is < 2^23 so the magic-number convert is valid.// The matching x lane was pre-divided by 16^k in load_vector.- const uint4 lanes = uint4(weight_words[i]) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);+ const uint4 lanes = uint4(ushort(weight_words[i] ^ packed_mask)) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);const U4 weight_vec4 = U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));accumulator += dot(x_vec4[i], weight_vec4);}} else if constexpr (BITS == 8) {using U4 = vec<U, 4>;- const device uint* weight_words = reinterpret_cast<const device uint*>(w);const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);- METAL_PRAGMA_UNROLL- for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {- // Mask each byte in place (no shifts); lane k value is b_k * 256^k. This- // exceeds the magic-number range for k=3, so use the hardware convert- // (exact for b_k * 256^k). x lane k was pre-divided by 256^k.- const uint4 lanes = uint4(weight_words[i]) & uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);- const U4 weight_vec4 = U4(float4(lanes));- accumulator += dot(x_vec4[i], weight_vec4);+ if (signed_codes) {+ const device char4* weight_vectors = reinterpret_cast<const device char4*>(w);+ METAL_PRAGMA_UNROLL+ for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {+ accumulator += dot(x_vec4[i], U4(weight_vectors[i]));+ }+ } else {+ const device uint* weight_words = reinterpret_cast<const device uint*>(w);+ METAL_PRAGMA_UNROLL+ for (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {+ // Keep each byte in place. Lane k is b_k * 256^k, while the+ // matching activation lane was pre-divided by 256^k.+ const uint4 lanes = uint4(weight_words[i]) & uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);+ accumulator += dot(x_vec4[i], U4(float4(lanes)));+ }}}- return scale * accumulator + sum * bias;+ const U adjusted_bias = (BITS == 8 && signed_codes) ? bias + scale * U(128) : bias;+ return scale * accumulator + sum * adjusted_bias;}template <typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U qdot_safe(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, int N) {+METAL_FUNC U+qdot_safe(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, int N, const bool signed_codes) {static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");U accumulator = 0;@@ -89,17 +99,18 @@ METAL_FUNC U qdot_safe(const device uint8_t* w, const thread U* x_thread, U scalusing U4 = vec<U, 4>;const device uint16_t* weight_words = reinterpret_cast<const device uint16_t*>(w);const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);+ const uint16_t packed_mask = signed_codes ? 0x8888u : 0u;int full_chunks = N / 4;for (int i = 0; i < full_chunks; i++) {- uint16_t weight_word = weight_words[i];+ uint16_t weight_word = weight_words[i] ^ packed_mask;U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(weight_word, weight_word >> 4, weight_word >> 8, weight_word >> 12));accumulator += dot(x_vec4[i], weight_vec4);}int remainder = N & 3;if (remainder > 0) {- uint16_t weight_word = weight_words[full_chunks];+ uint16_t weight_word = weight_words[full_chunks] ^ packed_mask;int base_index = 4 * full_chunks;accumulator += x_thread[base_index] * uint_to_fp<U>(weight_word & 0xf);if (remainder > 1)@@ -108,12 +119,20 @@ METAL_FUNC U qdot_safe(const device uint8_t* w, const thread U* x_thread, U scalaccumulator += x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);}} else if constexpr (BITS == 8) {- for (int i = 0; i < N; i++) {- accumulator += x_thread[i] * w[i];+ if (signed_codes) {+ const device int8_t* signed_weights = reinterpret_cast<const device int8_t*>(w);+ for (int i = 0; i < N; i++) {+ accumulator += x_thread[i] * U(signed_weights[i]);+ }+ } else {+ for (int i = 0; i < N; i++) {+ accumulator += x_thread[i] * U(w[i]);+ }}}- return scale * accumulator + sum * bias;+ const U adjusted_bias = (BITS == 8 && signed_codes) ? bias + scale * U(128) : bias;+ return scale * accumulator + sum * adjusted_bias;}} // namespace gemmdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 20c89be17..e449c7b08 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -5,7 +5,7 @@#include "../../../hadamard_transform/hadamard_transform.h"#include "../../common/fragment.h"#include "gemm_rht.h"-#include "../../common/mxu_fragment_ops.h"+#include "../../common/mxu_fragment/ops.h"#include "../../common/mxu_gemm_loop.h"#include "../../../generated/matmul.h"#include "../generated/gemm.h"@@ -215,7 +215,7 @@ struct MxuMmaCore {const ushort packed = *reinterpret_cast<const device ushort*>(b_packed_simdgroup + int(row) * b_row_stride_bytes + (k_base >> 1));- codes = unpack_nibbles_to_int8(uint(packed));+ codes = unpack_signed_nibbles_to_int8(uint(packed));}weight_vector[element_base + 0] = codes.x;weight_vector[element_base + 1] = codes.y;@@ -234,11 +234,6 @@ struct MxuMmaCore {right_src = right_src.bounded(simdgroup_limit_n, SIMDGROUP_BLOCK_K);}right_tile.load_from(simd_lane_id, right_src);- thread int8_t* right_codes = right_tile.elements();- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < right_tile.ELEMENTS_PER_FRAGMENT; ++i) {- right_codes[i] = unbias_uint8_to_int8(as_type<uchar>(right_codes[i]));- }}return right_tile;}@@ -404,15 +399,24 @@ struct MxuMmaCore {);uzu::matmul::Fragment<int, TILES_M, TILES_N, Ops> chunk_products;- auto right_tile = load_int8_weight_tile<ALIGNED_N>(- b_packed_simdgroup,- k_element_offset,- b_row_stride_bytes,- simdgroup_limit_n,- position,- thread_context.simd_lane_id- );- Ops::template fragment_mm<false, true>(chunk_products, activation_tile, right_tile);+ if constexpr (BITS == 4 && ALIGNED_N) {+ Ops::template fragment_matmul_int8_device_weights<metal::int4b_format>(+ chunk_products,+ activation_tile,+ b_packed_simdgroup + (k_element_offset >> 1),+ b_row_stride_bytes+ );+ } else {+ auto right_tile = load_int8_weight_tile<ALIGNED_N>(+ b_packed_simdgroup,+ k_element_offset,+ b_row_stride_bytes,+ simdgroup_limit_n,+ position,+ thread_context.simd_lane_id+ );+ Ops::template fragment_mm<false, true>(chunk_products, activation_tile, right_tile);+ }const uint act_group_index = k_offset_act_groups + uint(weight_group * act_chunks_per_weight_group + act_chunk);ActivationLineCache activation_scales;@@ -475,6 +479,7 @@ struct MxuMmaCore {const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,GemmDTransform output_transform,+ const bool signed_codes,const device BT* scales,const device BT* biases,const device uint8_t* zero_points,@@ -624,6 +629,7 @@ struct MxuMmaCore {weights_block,scales_offset,biases_offset,+ signed_codes,k_elements,b_shared,thread_context.simdgroup_index,@@ -638,6 +644,7 @@ struct MxuMmaCore {weights_block,scales_offset,zero_points_row_start,+ signed_codes,k_elements,groups_per_row,b_shared,@@ -649,6 +656,7 @@ struct MxuMmaCore {weights_block,scales_offset,nullptr,+ signed_codes,k_elements,groups_per_row,b_shared,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hindex da377d270..39e1b349f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -48,11 +48,13 @@ struct QuantizedBlockLoaderScaleBias {const device uint8_t* src;const device T* scales;const device T* biases;+ const bool signed_codes;QuantizedBlockLoaderScaleBias(const device uint8_t* src_,const device T* scales_,const device T* biases_,+ const bool signed_codes_,const int src_leading_dim_,threadgroup T* dst_,ushort simd_group_id [[simdgroup_index_in_threadgroup]],@@ -70,7 +72,7 @@ struct QuantizedBlockLoaderScaleBias {dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * pack_factor),src(src_ + tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor + tile_col_index * bytes_per_pack),scales(scales_ + tile_row_index * src_leading_dim_ / GROUP_SIZE),- biases(biases_ + tile_row_index * src_leading_dim_ / GROUP_SIZE) {}+ biases(biases_ + tile_row_index * src_leading_dim_ / GROUP_SIZE), signed_codes(signed_codes_) {}void load_unsafe() const {if constexpr (TILE_HAS_IDLE_THREADS) {@@ -82,7 +84,7 @@ struct QuantizedBlockLoaderScaleBias {T scale = *scales;T bias = *biases;for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);}}@@ -112,7 +114,7 @@ struct QuantizedBlockLoaderScaleBias {T scale = *scales;T bias = *biases;for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 42e1abd4b..981197036 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -52,11 +52,13 @@ struct QuantizedBlockLoaderScaleZeroPoint {const device T* scales;const device T* scales_row_start;const device uint8_t* zero_points_row_start;+ const bool signed_codes;QuantizedBlockLoaderScaleZeroPoint(const device uint8_t* src_,const device T* scales_,const device uint8_t* zero_points_row_start_,+ const bool signed_codes_,const int src_leading_dim_,const int groups_per_row_,threadgroup T* dst_,@@ -82,7 +84,8 @@ struct QuantizedBlockLoaderScaleZeroPoint {: (REDUCTION_DIMENSION == 1 ? (zero_points_row_start_ +tile_row_index * zero_point_row_stride<ushort(BITS)>(groups_per_row_)): zero_points_row_start_)- ) {}+ ),+ signed_codes(signed_codes_) {}inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {uint zero_point_value;@@ -115,7 +118,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {T bias;current_scale_bias(scale, bias);for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);}}@@ -143,7 +146,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {for (int i = 0; i < READS_PER_THREAD; i++) {int pack_index = tile_col_index + i;if (pack_index < valid_packs) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);if (pack_index == valid_packs - 1) {int remaining = valid_cols - pack_index * pack_factor;@@ -172,7 +175,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {T bias;current_scale_bias(scale, bias);for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.hindex 559c95d9e..9b1677df9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h@@ -71,49 +71,48 @@ METAL_FUNC uint decode_zero_point(const device uint8_t* zero_points_row, uint gr}}-METAL_FUNC char4 unpack_nibbles_to_int8(uint packed) {+METAL_FUNC char4 unpack_signed_nibbles_to_int8(uint packed) {uint spread = (packed | (packed << 8)) & 0x00FF00FFu;spread = (spread | (spread << 4)) & 0x0F0F0F0Fu;- return as_type<char4>(spread) - char4(char(symmetric_zero_point<4>()));-}--METAL_FUNC int8_t unbias_uint8_to_int8(uchar code) {- return as_type<int8_t>(uchar(code ^ uchar(symmetric_zero_point<8>())));+ constexpr uint sign_bits = symmetric_zero_point<4>() * 0x01010101u;+ return as_type<char4>(spread ^ sign_bits) - char4(char(symmetric_zero_point<4>()));}template <typename U, int N, int bits>-inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {+inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local, const bool signed_codes) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");- if (bits == 4) {+ if constexpr (bits == 4) {U s0 = scale;U s1 = scale / static_cast<U>(16.0f);- for (int i = 0; i < (N / 2); i++) {- w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;- w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;+ // Keep the mask a literal in each arm; a value derived from the function+ // constant inside the loop defeats vectorization of the unpack.+ if (signed_codes) {+ for (int i = 0; i < (N / 2); i++) {+ const uint8_t word = w[i] ^ uint8_t(0x88u);+ w_local[2 * i] = s0 * (word & 0x0f) + bias;+ w_local[2 * i + 1] = s1 * (word & 0xf0) + bias;+ }+ } else {+ for (int i = 0; i < (N / 2); i++) {+ w_local[2 * i] = s0 * (w[i] & 0x0f) + bias;+ w_local[2 * i + 1] = s1 * (w[i] & 0xf0) + bias;+ }}- } else if (bits == 8) {- for (int i = 0; i < N; i++) {- w_local[i] = scale * w[i] + bias;+ } else if constexpr (bits == 8) {+ if (signed_codes) {+ const device int8_t* signed_weights = reinterpret_cast<const device int8_t*>(w);+ const U adjusted_bias = bias + scale * U(128);+ for (int i = 0; i < N; i++) {+ w_local[i] = scale * U(signed_weights[i]) + adjusted_bias;+ }+ } else {+ for (int i = 0; i < N; i++) {+ w_local[i] = scale * U(w[i]) + bias;+ }}}}-template <>-inline void dequantize<bfloat, 8, 4>(const device uint8_t* w, bfloat scale, bfloat bias, threadgroup bfloat* w_local) {- const uint32_t packed = *reinterpret_cast<const device uint32_t*>(w);- const bfloat4 lo = bfloat4(as_type<uchar4>(packed & 0x0f0f0f0fu)) * scale + bias;- const bfloat4 hi = bfloat4(as_type<uchar4>(packed & 0xf0f0f0f0u)) * (scale * bfloat(0.0625f)) + bias;-- w_local[0] = lo.x;- w_local[1] = hi.x;- w_local[2] = lo.y;- w_local[3] = hi.y;- w_local[4] = lo.z;- w_local[5] = hi.z;- w_local[6] = lo.w;- w_local[7] = hi.w;-}-} // namespace gemm} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex bd1870521..86c405de5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -202,6 +202,7 @@ struct SimdgroupMmaCore {const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,GemmDTransform output_transform,+ const bool signed_codes,const device BT* scales,const device BT* biases,const device uint8_t* zero_points,@@ -273,6 +274,7 @@ struct SimdgroupMmaCore {weights_block,scales_offset,biases_offset,+ signed_codes,k_elements,b_shared,thread_context.simdgroup_index,@@ -286,6 +288,7 @@ struct SimdgroupMmaCore {weights_block,scales_offset,zero_points_row_start,+ signed_codes,k_elements,groups_per_row,b_shared,@@ -297,6 +300,7 @@ struct SimdgroupMmaCore {weights_block,scales_offset,nullptr,+ signed_codes,k_elements,groups_per_row,b_shared,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex d43e78977..c8da2bb8e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -122,6 +122,7 @@ KERNEL(Gemm)(const constant uint& group_count_z,const GemmDTransform output_transform SPECIALIZE,const GemmAlignment alignment SPECIALIZE,+ const bool signed_codes SPECIALIZE,threadgroup AT a_shared[GEMM_TGA_ELEMENTS],threadgroup BT b_shared[GEMM_TGB_ELEMENTS],const uint group_x GROUPS(group_count_x),@@ -147,6 +148,7 @@ KERNEL(Gemm)(params,alignment,output_transform,+ signed_codes,scales,biases,zero_points,@@ -166,6 +168,7 @@ KERNEL(Gemm)(params,alignment,output_transform,+ signed_codes,scales,biases,zero_points,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 6afe3efce..0f6a09979 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -4,14 +4,14 @@ use super::specialization::GemmSpecialization;use crate::{backends::{common::{- Allocation, Backend, BufferArg, Encoder,+ Allocation, BufferArg, Encoder,gpu_types::{- GemmParams, HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder,+ GemmParams, HADAMARD_TRANSFORM_BLOCK_SIZE,gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},},kernel::{- HadamardTransformKernel, Kernels, TensorAddBiasKernel,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError},+ ActivationTransform, TensorAddBiasKernel,+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulShape},},},metal::{@@ -40,7 +40,7 @@ pub struct GemmKernel {output_data_type: DataType,kernels: HashMap<GemmSpecialization, GemmMetalKernel>,pub bias_add: TensorAddBiasMetalKernel,- pub hadamard: <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel,+ output_rht: ActivationTransform<Metal>,split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,}@@ -52,18 +52,14 @@ impl GemmKernel {output_data_type: DataType,) -> Result<Self, MetalError> {let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true, false)?;- let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(- context,- output_data_type,- HadamardTransformOrder::Output,- )?;+ let output_rht = ActivationTransform::output_rht(context, output_data_type, true)?;let kernel = Self {weights_data_type,input_data_type,output_data_type,kernels: HashMap::new(),bias_add,- hadamard,+ output_rht,split_k_reduce: HashMap::new(),};Ok(kernel)@@ -91,6 +87,7 @@ impl GemmKernel {specialization.a_prologue,specialization.output_transform,specialization.alignment,+ specialization.signed_codes,)?;Ok(entry.insert(kernel))},@@ -111,29 +108,25 @@ impl GemmKernel {}}- pub(crate) fn should_skip_gemv_for_mxu<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(+ pub(crate) fn should_skip_gemv_for_mxu(&self,- arguments: &MatmulArguments<'a, 'b, 'd, Metal, TB>,+ shape: &MatmulShape,) -> bool {- if arguments.gather_indices.is_some() {+ if shape.gathered {// TODO: gathered GEMMreturn false;}- match (- arguments.m,- arguments.n == arguments.k,- (self.weights_data_type, self.input_data_type, self.output_data_type),- ) {+ match (shape.m, shape.n == shape.k, (self.weights_data_type, self.input_data_type, self.output_data_type)) {(4, true, (DataType::F32, DataType::F32, DataType::F32))| (5, _, (DataType::BF16, DataType::BF16, DataType::BF16)) => return false,_ => {},}- match arguments.m {+ match shape.m {0..=3 => return false,4 => {// The M4 MXU tile only uses a quarter of its rows; avoid it for wide-N shapes.- let small_enough_for_mxu = arguments.n <= 6144 && arguments.k <= 9728;- let k_dominates = arguments.k > 3_u32.saturating_mul(arguments.n);+ let small_enough_for_mxu = shape.n <= 6144 && shape.k <= 9728;+ let k_dominates = shape.k > 3_u32.saturating_mul(shape.n);if !(small_enough_for_mxu || k_dominates) {return false;}@@ -141,15 +134,19 @@ impl GemmKernel {_ => {},}matches!(- self.select_mxu_tiling(arguments),+ self.mxu_tiling_for(shape),Some(GemmTiling::Tile16x32x256_Simdgroups1x1 | GemmTiling::Tile16x128x256_Simdgroups1x4))}- fn select_mxu_tiling<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(+ /// Int8 activations always have an MXU tiling, so callers short-circuit before asking.+ fn mxu_tiling_for(&self,- arguments: &MatmulArguments<'a, 'b, 'd, Metal, TB>,+ shape: &MatmulShape,) -> Option<GemmTiling> {+ if !shape.a_full_precision {+ return None;+ }if ![self.weights_data_type, self.input_data_type, self.output_data_type].into_iter().all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32))@@ -157,34 +154,18 @@ impl GemmKernel {return None;}- match &arguments.b {- MatmulB::FullPrecision {- ..- } => Some(if arguments.b_transpose {- select_mxu_tiling(arguments.m, arguments.n, arguments.k)+ match shape.b_prologue {+ GemmBPrologueKind::FullPrecision => Some(if shape.b_transpose {+ mxu_tiling(shape.m, shape.n, shape.k)} else {- select_base_mxu_tiling(arguments.m, arguments.n)+ mxu_tiling_by_mn(shape.m, shape.n)}),- MatmulB::ScaleBiasDequant {- ..- }- | MatmulB::ScaleZeroPointDequant {- ..- }- | MatmulB::ScaleSymmetricDequant {- ..- } => {- if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {+ _ => {+ if !shape.b_transpose || shape.b_leading_dimension.is_some() {return None;}- let group_size = arguments.b.group_size().unwrap_or(0);- let int8_activations = arguments.a.prologue_kind() == GemmAPrologueKind::Int8Symmetric;- let tiling =- select_mxu_quant_tiling(arguments.m, arguments.n, arguments.k, group_size, int8_activations);- if int8_activations {- return (group_size != 0 && arguments.k.is_multiple_of(group_size)).then_some(tiling);- }- arguments.k.is_multiple_of(tiling.block_k()).then_some(tiling)+ let tiling = mxu_quant_tiling(shape.m, shape.n, shape.k, shape.b_group_size.unwrap_or(0), false);+ shape.k.is_multiple_of(tiling.block_k()).then_some(tiling)},}}@@ -194,9 +175,9 @@ impl GemmKernel {arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {+ let int8_activations = arguments.a.prologue_kind() == GemmAPrologueKind::Int8Symmetric;let path = if encoder.context().device.supports_mxu()- && (arguments.a.prologue_kind() == GemmAPrologueKind::Int8Symmetric- || self.select_mxu_tiling(&arguments).is_some())+ && (int8_activations || self.mxu_tiling_for(&MatmulShape::from_arguments(&arguments)).is_some()){GemmDispatchPath::Mxu} else {@@ -253,6 +234,7 @@ impl GemmKernel {let b_prologue = arguments.b.b_prologue();let bits_per_b = arguments.b.bits_per_b();let group_size = arguments.b.group_size();+ let weights_signed_codes = arguments.b.signed_codes();let MatmulArguments {a,@@ -286,12 +268,12 @@ impl GemmKernel {let tiling = if use_mxu {if b_transpose {- select_mxu_tiling(m, n, k)+ mxu_tiling(m, n, k)} else {- select_base_mxu_tiling(m, n)+ mxu_tiling_by_mn(m, n)}} else {- select_simdgroup_tiling(m, n, k)+ simdgroup_tiling(m, n, k)};let threadgroups_per_row = n.div_ceil(tiling.block_n());@@ -341,6 +323,7 @@ impl GemmKernel {b_prologue,bits_per_b,group_size,+ false,split_k,output_transform,output_bias,@@ -380,6 +363,7 @@ impl GemmKernel {bits_per_b,group_size,a_prologue: GemmAPrologueKind::FullPrecision,+ signed_codes: false,};specialization.validate()?;let kernel = self.get_or_create(encoder.context(), specialization)?;@@ -444,7 +428,14 @@ impl GemmKernel {scales: activation_scales,group_sums: activation_group_sums,} => {- validate_int8_activation_arguments(use_mxu, k, b_prologue, bits_per_b, group_size)?;+ validate_int8_activation_arguments(+ use_mxu,+ weights_signed_codes,+ k,+ b_prologue,+ bits_per_b,+ group_size,+ )?;if output_transform.contains(GemmDTransform::SOFT_CAP) {return Err(MatmulError::UnsupportedDOp {bit: GemmDTransform::SOFT_CAP,@@ -464,9 +455,9 @@ impl GemmKernel {};let tiling = if use_mxu {- select_mxu_quant_tiling(m, n, k, group_size.unwrap_or(0), a_is_int8)+ mxu_quant_tiling(m, n, k, group_size.unwrap_or(0), a_is_int8)} else {- select_quant_tiling(m, n, group_size.unwrap_or(0))+ simdgroup_quant_tiling(m, n, group_size.unwrap_or(0))};let alignment =GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);@@ -514,6 +505,7 @@ impl GemmKernel {b_prologue,bits_per_b,group_size,+ weights_signed_codes,split_k,output_transform,output_bias,@@ -532,6 +524,7 @@ impl GemmKernel {bits_per_b,group_size,a_prologue,+ signed_codes: weights_signed_codes,};specialization.validate()?;let kernel = self.get_or_create(encoder.context(), specialization)?;@@ -595,6 +588,7 @@ impl GemmKernel {b_prologue: GemmBPrologueKind,bits_per_b: Option<u32>,group_size: Option<u32>,+ signed_codes: bool,split_k: u32,output_transform: GemmDTransform,output_bias: Option<&Allocation<Metal>>,@@ -619,6 +613,7 @@ impl GemmKernel {bits_per_b,group_size,a_prologue,+ signed_codes,};part_spec.validate()?;@@ -679,7 +674,7 @@ impl GemmKernel {if output_transform.contains(GemmDTransform::RHT)&& let Some(factors) = rht_factors{- self.hadamard.encode(&mut *d, factors, n, m, encoder);+ self.output_rht.encode_fp_in_place(&mut *d, factors, m, n, encoder);}Ok(())}@@ -687,6 +682,7 @@ impl GemmKernel {fn validate_int8_activation_arguments(use_mxu: bool,+ weights_signed_codes: bool,k: u32,b_prologue: GemmBPrologueKind,bits_per_b: Option<u32>,@@ -694,6 +690,7 @@ fn validate_int8_activation_arguments() -> Result<(), MetalError> {let weight_gs_ok = matches!(weight_group_size, Some(32 | 64 | 128));let compatible = use_mxu+ && weights_signed_codes&& matches!(b_prologue,GemmBPrologueKind::ScaleSymmetricDequant@@ -707,7 +704,7 @@ fn validate_int8_activation_arguments(if !compatible {return Err(MatmulError::IncompatibleA {path: "Gemm",- reason: "symmetric int8 activations require MXU and unsigned 4/8-bit quantized weights with group size 32/64/128",+ reason: "symmetric int8 activations require MXU and signed 4/8-bit quantized weight codes with group size 32/64/128",}.into());}@@ -802,7 +799,7 @@ fn select_split_k(split_k}-pub(crate) fn select_simdgroup_tiling(+pub(crate) fn simdgroup_tiling(m: u32,n: u32,k: u32,@@ -814,7 +811,7 @@ pub(crate) fn select_simdgroup_tiling(}}-pub(crate) fn select_mxu_tiling(+pub(crate) fn mxu_tiling(m: u32,n: u32,k: u32,@@ -828,15 +825,15 @@ pub(crate) fn select_mxu_tiling(};}return if m < 16 {- select_small_m_mxu_tiling(n, k)+ mxu_tiling_small_m(n, k)} else {- select_base_mxu_tiling(m, n)+ mxu_tiling_by_mn(m, n)};}- select_base_mxu_tiling(m, n)+ mxu_tiling_by_mn(m, n)}-fn select_base_mxu_tiling(+fn mxu_tiling_by_mn(m: u32,n: u32,) -> GemmTiling {@@ -851,7 +848,7 @@ fn select_base_mxu_tiling(}}-fn select_small_m_mxu_tiling(+fn mxu_tiling_small_m(n: u32,k: u32,) -> GemmTiling {@@ -867,7 +864,7 @@ fn select_small_m_mxu_tiling(GemmTiling::Tile32x64x256_Simdgroups2x2}-pub(crate) fn select_mxu_quant_tiling(+pub(crate) fn mxu_quant_tiling(m: u32,n: u32,k: u32,@@ -875,9 +872,9 @@ pub(crate) fn select_mxu_quant_tiling(int8_activations: bool,) -> GemmTiling {let tiling = if int8_activations {- select_mxu_tiling(m, n, k)+ mxu_tiling(m, n, k)} else {- select_base_mxu_tiling(m, n)+ mxu_tiling_by_mn(m, n)};if tiling.fits_quant_group_size(group_size) {tiling@@ -886,7 +883,7 @@ pub(crate) fn select_mxu_quant_tiling(}}-pub(crate) fn select_quant_tiling(+pub(crate) fn simdgroup_quant_tiling(m: u32,n: u32,group_size: u32,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 0a311a97b..e90e15375 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -18,6 +18,7 @@ pub(super) struct GemmSpecialization {pub(super) b_prologue: GemmBPrologueKind,pub(super) bits_per_b: Option<u32>,pub(super) group_size: Option<u32>,+ pub(super) signed_codes: bool,}impl GemmSpecialization {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.hindex 5a185b6e8..0c66e9157 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h@@ -32,7 +32,8 @@ struct BSource {uint out_row,uint batch_idx,uint simd_lane,- uint k_slice+ uint k_slice,+ const bool signed_codes) {if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {FullPrecisionBSource<BT, AT, U, RESULTS_PER_SIMDGROUP, K_SPLIT, INPUT_ALIGNED>::accumulate(@@ -62,7 +63,8 @@ struct BSource {out_vec_size,out_row,batch_idx,- simd_lane+ simd_lane,+ signed_codes);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.hindex 3aa1639e0..cda4f41a3 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quantized_b_source.h@@ -30,7 +30,8 @@ struct QuantizedBSource {uint out_vec_size,uint out_row,uint batch_idx,- uint simd_lane+ uint simd_lane,+ const bool signed_codes) {constexpr uint pack_factor = get_pack_factor<BITS, 32>();constexpr uint bytes_per_pack = get_bytes_per_pack<BITS, 32>();@@ -58,7 +59,7 @@ struct QuantizedBSource {uint k = 0;for (; k + block_size <= in_vec_size; k += block_size) {- U input_sum = load_vector<AT, U, values_per_thread, BITS>(input, input_values);+ U input_sum = load_vector<AT, U, values_per_thread, BITS>(input, input_values, signed_codes);RowParams row_params;row_state.load(row_params, gather_indices, gathered, batch_idx, out_vec_size, out_row);@@ -71,7 +72,8 @@ struct QuantizedBSource {input_values,row_params.scale[row],row_params.offset[row],- input_sum+ input_sum,+ signed_codes);}@@ -100,7 +102,8 @@ struct QuantizedBSource {row_params.scale[row],row_params.offset[row],input_sum,- remaining+ remaining,+ signed_codes);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex e2d454d56..08de3feeb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -72,6 +72,7 @@ KERNEL(Gemv)(OPTIONAL(output_transform.contains(GemmDTransform::SOFT_CAP)),const GemmDTransform output_transform SPECIALIZE,const bool gathered SPECIALIZE,+ const bool signed_codes SPECIALIZE,threadgroup float shared_results[NUM_SIMDGROUPS * RESULTS_PER_SIMDGROUP],const uint batch_idx GROUPS(batch_size),const uint out_block_idx GROUPS(group_count_x),@@ -99,7 +100,8 @@ KERNEL(Gemv)(tile.out_row,batch_idx,simd_lane,- tile.k_slice+ tile.k_slice,+ signed_codes);Reduce<U, K_SPLIT, NUM_SIMDGROUPS, RESULTS_PER_SIMDGROUP>::run(diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 5f4c1df0e..95eb934e6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -12,7 +12,7 @@ use crate::{HADAMARD_TRANSFORM_BLOCK_SIZE,gemm::{GemmBPrologueKind, GemmDTransform},},- kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError},+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulShape},},metal::{Metal, context::MetalContext, device_tier::DeviceTier, kernel::GemvMetalKernel},},@@ -40,48 +40,50 @@ pub(crate) struct GemvSpecialization {results_per_simdgroup: u32,num_simdgroups: u32,gathered: bool,+ signed_codes: bool,}impl GemvSpecialization {- pub(crate) fn select<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(- args: &MatmulArguments<'a, 'b, 'd, Metal, TB>,+ pub(crate) fn select_shape(+ shape: &MatmulShape,weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,device_tier: DeviceTier,) -> Option<GemvSpecialization> {- if !args.b_transpose || !matches!(args.a, MatmulA::FullPrecision { .. }) {+ if !shape.b_transpose || !shape.a_full_precision {return None;}- let is_quant = !matches!(args.b, MatmulB::FullPrecision { .. });- let gathered = args.gather_indices.is_some();+ let is_quant = shape.is_quant();let bad_leading_dimension = if is_quant {- args.b_leading_dimension.is_some()+ shape.b_leading_dimension.is_some()} else {- args.b_leading_dimension.is_some_and(|ld| ld != args.k)+ shape.b_leading_dimension.is_some_and(|ld| ld != shape.k)};if bad_leading_dimension {return None;}- if args.d_transform.accumulate && !args.n.is_multiple_of(32) {+ if shape.d_transform.contains(GemmDTransform::ACCUMULATE) && !shape.n.is_multiple_of(32) {return None;}- if args.d_transform.rht_factors.is_some() && !args.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE as u32) {+ if shape.d_transform.contains(GemmDTransform::RHT)+ && !shape.n.is_multiple_of(HADAMARD_TRANSFORM_BLOCK_SIZE as u32)+ {return None;}if is_quant {- if args.n < DEFAULT_RESULTS_PER_SIMDGROUP || args.m >= 5 {+ if shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m >= 5 {return None;}} else {let mixed_precision = weights_data_type == DataType::F32&& (input_data_type != DataType::F32 || output_data_type != DataType::F32);- if mixed_precision || args.n < DEFAULT_RESULTS_PER_SIMDGROUP || args.m > max_gemv_batch_threshold() {+ if mixed_precision || shape.n < DEFAULT_RESULTS_PER_SIMDGROUP || shape.m > max_gemv_batch_threshold() {return None;}}- let bits = args.b.bits_per_b().unwrap_or(0);+ let bits = shape.b_bits.unwrap_or(0);let block_size = if !is_quant {FP_K_BLOCK} else if bits == 4 {@@ -89,28 +91,29 @@ impl GemvSpecialization {} else {256};- let input_aligned = args.k.is_multiple_of(block_size);- let has_rht = args.d_transform.rht_factors.is_some();+ let input_aligned = shape.k.is_multiple_of(block_size);+ let has_rht = shape.d_transform.contains(GemmDTransform::RHT);let bf16_io = input_data_type == DataType::BF16 && output_data_type == DataType::BF16;let tile = if is_quant && bf16_io {- policy::quant_tile(args.m, args.n, args.k, bits, has_rht, device_tier)+ policy::quant_tile(shape.m, shape.n, shape.k, bits, has_rht, device_tier)} else if is_quant || has_rht {// Non-bf16 quant IO and fp+RHT keep the default tile (the only// one instantiated for those modes).policy::DEFAULT_TILE} else {- policy::fp_tile(args.m, args.n, args.k, input_aligned, device_tier)+ policy::fp_tile(shape.m, shape.n, shape.k, input_aligned, device_tier)};Some(Self {- b_prologue: args.b.b_prologue(),- group_size: args.b.group_size().unwrap_or(0),+ b_prologue: shape.b_prologue,+ group_size: shape.b_group_size.unwrap_or(0),bits,- output_transform: args.d_transform.mask(),+ output_transform: shape.d_transform,input_aligned,k_split: tile.k_split,results_per_simdgroup: tile.results_per_simdgroup,num_simdgroups: tile.num_simdgroups,- gathered,+ gathered: shape.gathered,+ signed_codes: shape.signed_codes,})}}@@ -166,6 +169,7 @@ impl GemvDispatch {specialization.num_simdgroups,specialization.output_transform,specialization.gathered,+ specialization.signed_codes,).map_err(MatmulError::BackendError)?;Ok(entry.insert(kernel))diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex deed61a7e..8d9f05d9d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -7,7 +7,7 @@ use crate::{backends::{common::{BufferArg, Encoder,- kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulPath, MatmulShape},},metal::{Metal, context::MetalContext, error::MetalError, metal_extensions::DeviceExt},},@@ -22,6 +22,25 @@ pub struct MatmulMetalKernel {output_data_type: DataType,}+impl MatmulMetalKernel {+ fn gemv_specialization(+ &self,+ shape: &MatmulShape,+ context: &MetalContext,+ ) -> Option<GemvSpecialization> {+ if context.device.supports_mxu() && self.gemm.should_skip_gemv_for_mxu(shape) {+ return None;+ }+ GemvSpecialization::select_shape(+ shape,+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,+ context.device_tier(),+ )+ }+}+impl MatmulKernel for MatmulMetalKernel {type Backend = Metal;@@ -49,21 +68,25 @@ impl MatmulKernel for MatmulMetalKernel {})}+ fn select_path(+ &self,+ shape: &MatmulShape,+ context: &MetalContext,+ ) -> MatmulPath {+ if self.gemv_specialization(shape, context).is_some() {+ MatmulPath::Gemv+ } else {+ MatmulPath::Gemm+ }+ }+fn encode<'a, 'b, 'd, TB: BufferArg<'b, Metal>>(&mut self,arguments: MatmulArguments<'a, 'b, 'd, Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {- let skip_gemv = encoder.context().device.supports_mxu() && self.gemm.should_skip_gemv_for_mxu(&arguments);- if !skip_gemv- && let Some(gemv) = GemvSpecialization::select(- &arguments,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- encoder.context().device_tier(),- )- {+ let shape = MatmulShape::from_arguments(&arguments);+ if let Some(gemv) = self.gemv_specialization(&shape, encoder.context()) {return self.gemv.encode(arguments, gemv, encoder).map_err(MetalError::from);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/mod.rsindex 8b7b69956..f061464d8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/mod.rs@@ -28,7 +28,3 @@ impl Kernels for MetalKernels {type MatmulKernel = matmul::MatmulMetalKernel;type RadixTopKSmall = radix_top_k_small::MetalRadixTopKSmall;}--#[cfg(test)]-#[path = "../../../../tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs"]-mod rht_quantize_activations_test;diff --git a/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal b/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metaldeleted file mode 100644index 099f03488..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/rht_quantize_activations/rht_quantize_activations.metal+++ /dev/null@@ -1,49 +0,0 @@-#include <metal_stdlib>-#include "../common/defines.h"-#include "../common/dsl.h"-#include "../hadamard_transform/hadamard_transform.h"--using namespace metal;--UZU_CONST float SYM_QMAX = 127.0;--template <typename InputT>-VARIANTS(InputT, float, bfloat)-PUBLIC KERNEL(RHTQuantizeActivations)(- const device InputT* input,- device int8_t* q_out,- device float* scales_out,- device int32_t* group_sums_out OPTIONAL(emit_group_sums),- const device int32_t* rht_factors,- constant uint& batch_size,- constant uint& element_count,- constant uint& group_size,- const bool emit_group_sums SPECIALIZE,- uint block_index GROUPS(element_count.div_ceil(METAL_SIMD_SIZE)),- uint batch_index GROUPS(batch_size),- uint lane_index THREADS(METAL_SIMD_SIZE)-) {- const uint factor_index = block_index * METAL_SIMD_SIZE + lane_index;- const uint element_index = batch_index * element_count + factor_index;-- float value = static_cast<float>(input[element_index]);- value = simdgroup_input_random_hadamard_transform(lane_index, value, rht_factors[factor_index]);-- const float magnitude = max(fabs(simd_min(value)), fabs(simd_max(value)));- const float scale = isfinite(magnitude) && magnitude > 0.0f ? magnitude / SYM_QMAX : 1.0f;-- const int8_t code = static_cast<int8_t>(clamp(round(value / scale), -SYM_QMAX, SYM_QMAX));- q_out[element_index] = code;-- const uint group_index = batch_index * (element_count / group_size) + block_index;- if (lane_index == 0) {- scales_out[group_index] = scale;- }-- if (emit_group_sums) {- const int group_sum = simd_sum(int(code));- if (lane_index == 0) {- group_sums_out[group_index] = group_sum;- }- }-}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 2b5350cc4..2541d4742 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -5,9 +5,9 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Encoder, Kernels,- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder, QuantizationMethod, QuantizationMode},+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},kernel::{- FullPrecisionEmbeddingLookupKernel, HadamardTransformKernel, LogitSoftCapKernel,+ ActivationTransform, FullPrecisionEmbeddingLookupKernel, LogitSoftCapKernel,QuantizedEmbeddingLookupKernel,matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},@@ -92,7 +92,7 @@ enum UntiedEmbeddingReadoutType<B: Backend> {struct InputHadamard<B: Backend> {factors: Allocation<B>,- kernel: <B::Kernels as Kernels>::HadamardTransformKernel,+ kernel: ActivationTransform<B>,}enum EmbeddingTying<B: Backend> {@@ -480,12 +480,8 @@ impl<B: Backend> Embedding<B> {.leaf("input_signs")?.validate(&[model_dim as usize], DataType::I32)?.read_allocation()?;- let kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(- context,- data_type,- HadamardTransformOrder::Input,- )- .map_err(EmbeddingError::BackendError)?;+ let kernel = ActivationTransform::input_rht(context, data_type, false)+ .map_err(EmbeddingError::BackendError)?;let input_hadamard = Some(InputHadamard {factors,kernel,@@ -727,12 +723,12 @@ impl<B: Backend> Embedding<B> {Some(input_hadamard) => {let mut transformed =encoder.allocate_scratch(input_allocation.size()).map_err(EmbeddingError::BackendError)?;- encoder.encode_copy(input_allocation, .., &mut transformed, ..);- input_hadamard.kernel.encode(+ input_hadamard.kernel.encode_fp(+ input_allocation,&mut transformed,&input_hadamard.factors,- self.model_dim,batch_dim as u32,+ self.model_dim,encoder,);rht_input.insert(transformed)@@ -860,12 +856,12 @@ impl<B: Backend> Embedding<B> {let a = match input_hadamard {Some(input_hadamard) => {let mut transformed = encoder.allocate_scratch(input.size()).map_err(EmbeddingError::BackendError)?;- encoder.encode_copy(input, .., &mut transformed, ..);- input_hadamard.kernel.encode(+ input_hadamard.kernel.encode_fp(+ input,&mut transformed,&input_hadamard.factors,- self.model_dim,rows as u32,+ self.model_dim,encoder,);rht_input.insert(transformed)@@ -916,6 +912,7 @@ fn quantized_matmul_b<'a, B: Backend>(biases: zero_points_or_biases.expect("ScaleBias quantization requires biases"),mode: readout_config.mode,group_size: readout_config.group_size,+ signed_codes: false,},QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {b: weights,@@ -923,12 +920,14 @@ fn quantized_matmul_b<'a, B: Backend>(zero_points: zero_points_or_biases.expect("ScaleZeroPoint quantization requires zero_points"),mode: readout_config.mode,group_size: readout_config.group_size,+ signed_codes: false,},QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {b: weights,scales,mode: readout_config.mode,group_size: readout_config.group_size,+ signed_codes: false,},}}diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex f2f97d943..38e54e978 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -8,7 +8,7 @@ use crate::{gpu_types::{QuantizationMethod, QuantizationMode},kernel::{Kernels,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulPath, MatmulShape},},},config::weight_matrix::{AnyWeightMatrixSpec, Layout, int_spec::IntSpec, mlx_spec::MLXSpec},@@ -38,6 +38,7 @@ enum Mode<B: Backend> {scales: Allocation<B>,zero_points_or_biases: Option<Allocation<B>>,output_hadamard_factors: Option<Allocation<B>>,+ signed_codes: bool,},}@@ -187,6 +188,7 @@ impl<B: Backend> LinearMatmul<B> {scales,zero_points_or_biases,output_hadamard_factors,+ signed_codes: false,},})}@@ -209,6 +211,28 @@ fn load_biases<B: Backend>(}impl<B: Backend> LinearMatmul<B> {+ pub(super) fn make_weight_codes_signed(&mut self) {+ let Mode::Quantized {+ mode,+ signed_codes,+ ..+ } = &mut self.mode+ else {+ return;+ };+ if *signed_codes {+ return;+ }+ let Some(sign_flip_mask) = mode.weight_codes_sign_flip_mask() else {+ return;+ };+ let broadcast_mask = u64::from(sign_flip_mask) * 0x0101_0101_0101_0101;+ let (prefix, words, suffix) = bytemuck::pod_align_to_mut::<u8, u64>(self.weights.as_slice_mut());+ words.iter_mut().for_each(|word| *word ^= broadcast_mask);+ prefix.iter_mut().chain(suffix.iter_mut()).for_each(|code| *code ^= sign_flip_mask);+ *signed_codes = true;+ }+pub(super) fn encode_with_a(&self,a: MatmulA<'_, B>,@@ -218,7 +242,27 @@ impl<B: Backend> LinearMatmul<B> {let mut output =encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;- let b = match &self.mode {+ self.kernel.lock().encode(+ MatmulArguments {+ a,+ b: self.matmul_b(),+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut output,+ d_transform: self.d_ops(),+ gather_indices: None,+ m: batch_dim as u32,+ n: self.output_dim as u32,+ k: self.input_dim as u32,+ },+ encoder,+ )?;++ Ok(output)+ }++ fn matmul_b(&self) -> MatmulB<'_, B> {+ match &self.mode {Mode::FullPrecision => MatmulB::FullPrecision {b: &self.weights,},@@ -228,6 +272,7 @@ impl<B: Backend> LinearMatmul<B> {group_size,scales,zero_points_or_biases,+ signed_codes,..} => match method {QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {@@ -236,6 +281,7 @@ impl<B: Backend> LinearMatmul<B> {biases: zero_points_or_biases.as_ref().expect("ScaleBias quantization requires biases"),mode: *mode,group_size: *group_size,+ signed_codes: *signed_codes,},QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {b: &self.weights,@@ -245,46 +291,54 @@ impl<B: Backend> LinearMatmul<B> {.expect("ScaleZeroPoint quantization requires zero_points"),mode: *mode,group_size: *group_size,+ signed_codes: *signed_codes,},QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {b: &self.weights,scales,mode: *mode,group_size: *group_size,+ signed_codes: *signed_codes,},},- };+ }+ }- let rht_factors = match &self.mode {- Mode::Quantized {- output_hadamard_factors: Some(factors),- ..- } => Some(factors),- _ => None,- };- let d_transform = MatmulDOps {+ fn d_ops(&self) -> MatmulDOps<'_, B> {+ MatmulDOps {bias: self.biases.as_ref(),- rht_factors,- ..MatmulDOps::none()- };-- self.kernel.lock().encode(- MatmulArguments {- a,- b,- b_leading_dimension: None,- b_transpose: true,- d: &mut output,- d_transform,- gather_indices: None,- m: batch_dim as u32,- n: self.output_dim as u32,- k: self.input_dim as u32,+ rht_factors: match &self.mode {+ Mode::Quantized {+ output_hadamard_factors: Some(factors),+ ..+ } => Some(factors),+ _ => None,},- encoder,- )?;+ ..MatmulDOps::none()+ }+ }- Ok(output)+ pub(super) fn select_path(+ &self,+ batch_dim: usize,+ context: &B::Context,+ ) -> MatmulPath {+ let b = self.matmul_b();+ let shape = MatmulShape {+ m: batch_dim as u32,+ n: self.output_dim as u32,+ k: self.input_dim as u32,+ b_transpose: true,+ b_leading_dimension: None,+ b_prologue: b.b_prologue(),+ b_bits: b.bits_per_b(),+ b_group_size: b.group_size(),+ signed_codes: b.signed_codes(),+ a_full_precision: true,+ gathered: false,+ d_transform: self.d_ops().mask(),+ };+ self.kernel.lock().select_path(&shape, context)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex 48508a8bc..c27e02963 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -5,9 +5,9 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Encoder,- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,kernel::{- HadamardTransformKernel, Kernels,+ ActivationTransform, Kernels,matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},@@ -31,8 +31,8 @@ pub enum QLoRALinearWrapperError<B: Backend> {pub struct QLoRALinearWrapper<B: Backend> {base_linear: LinearMatmul<B>,- input_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,- output_hadamard: Option<(<B::Kernels as Kernels>::HadamardTransformKernel, Allocation<B>)>,+ input_hadamard: Option<(ActivationTransform<B>, Allocation<B>)>,+ output_hadamard: Option<(ActivationTransform<B>, Allocation<B>)>,adapter_down_kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,adapter_up_kernel: Mutex<<B::Kernels as Kernels>::MatmulKernel>,adapter_down: Allocation<B>,@@ -93,21 +93,13 @@ impl<B: Backend> QLoRALinearWrapper<B> {.read_allocation()?;(Some((- <B::Kernels as Kernels>::HadamardTransformKernel::new(- context,- input_data_type,- HadamardTransformOrder::Input,- )- .map_err(QLoRALinearWrapperError::BackendError)?,+ ActivationTransform::input_rht(context, input_data_type, false)+ .map_err(QLoRALinearWrapperError::BackendError)?,input_factors,)),Some((- <B::Kernels as Kernels>::HadamardTransformKernel::new(- context,- output_data_type,- HadamardTransformOrder::Output,- )- .map_err(QLoRALinearWrapperError::BackendError)?,+ ActivationTransform::output_rht(context, output_data_type, true)+ .map_err(QLoRALinearWrapperError::BackendError)?,output_factors,)),)@@ -198,12 +190,12 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {let base_input = if let Some((input_hadamard_kernel, input_factors)) = &self.input_hadamard {let mut base_input =encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dim], self.input_data_type))?;- encoder.encode_copy(&input, .., &mut base_input, ..);- input_hadamard_kernel.encode(+ input_hadamard_kernel.encode_fp(+ &input,&mut base_input,input_factors,- self.input_dim as u32,batch_dim as u32,+ self.input_dim as u32,encoder,);base_input@@ -241,11 +233,11 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {}if let Some((output_hadamard_kernel, output_factors)) = &self.output_hadamard {- output_hadamard_kernel.encode(+ output_hadamard_kernel.encode_fp_in_place(&mut output,output_factors,- self.output_dim as u32,batch_dim as u32,+ self.output_dim as u32,encoder,);}diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex 12e51b498..9058603aa 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -4,8 +4,11 @@ use crate::{array::size_for_shape,backends::common::{Allocation, Backend, Context, DeviceCapabilities, Encoder,- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder},- kernel::{HadamardTransformKernel, Kernels, RHTQuantizeActivationsKernel, matmul::MatmulA},+ gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,+ kernel::{+ ActivationTransform,+ matmul::{MatmulA, MatmulPath},+ },},config::weight_matrix::{AnyWeightMatrixSpec, Layout,@@ -81,14 +84,9 @@ fn weights_need_group_sums(quantization_spec: &AnyWeightMatrixSpec) -> bool {)}-struct SymmetricInt8Preparation<B: Backend> {- kernel: <B::Kernels as Kernels>::RHTQuantizeActivationsKernel,- emit_group_sums: bool,-}-pub struct RHTLinearWrapper<B: Backend> {- input_hadamard_kernel: <B::Kernels as Kernels>::HadamardTransformKernel,- symmetric_int8_preparation: Option<SymmetricInt8Preparation<B>>,+ input_transform: ActivationTransform<B>,+ quantize_transform: Option<ActivationTransform<B>>,input_factors: Allocation<B>,inner_linear: LinearMatmul<B>,input_dimension: usize,@@ -128,14 +126,10 @@ impl<B: Backend> RHTLinearWrapper<B> {let quantized_weights_tree = weights_tree.subtree("quantized")?;let quantization_spec = quantized_weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;- let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(- context,- input_data_type,- HadamardTransformOrder::Input,- )- .map_err(RHTLinearWrapperError::BackendError)?;+ let input_transform = ActivationTransform::input_rht(context, input_data_type, true)+ .map_err(RHTLinearWrapperError::BackendError)?;- let symmetric_int8_preparation = if int8_activations_eligible::<B>(+ let quantize_transform = if int8_activations_eligible::<B>(context,&quantization_spec,input_dimension,@@ -144,18 +138,14 @@ impl<B: Backend> RHTLinearWrapper<B> {) {let emit_group_sums = weights_need_group_sums(&quantization_spec);Some(- <B::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(context, input_data_type, emit_group_sums)- .map(|kernel| SymmetricInt8Preparation {- kernel,- emit_group_sums,- })+ ActivationTransform::quantize(context, input_data_type, emit_group_sums).map_err(RHTLinearWrapperError::BackendError)?,)} else {None};- let inner_linear = LinearMatmul::quantized(+ let mut inner_linear = LinearMatmul::quantized(context,quantization_spec,input_dimension,@@ -167,10 +157,13 @@ impl<B: Backend> RHTLinearWrapper<B> {has_biases.then_some(parameter_tree),Some(output_factors),)?;+ if quantize_transform.is_some() {+ inner_linear.make_weight_codes_signed();+ }Ok(Self {- input_hadamard_kernel,- symmetric_int8_preparation,+ input_transform,+ quantize_transform,input_factors,inner_linear,input_dimension,@@ -185,17 +178,19 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- if let Some(preparation) = &self.symmetric_int8_preparation {+ if let Some(quantize_transform) = &self.quantize_transform+ && self.inner_linear.select_path(batch_dim, encoder.context()) == MatmulPath::Gemm+ {let groups_per_row = self.input_dimension.div_ceil(HADAMARD_TRANSFORM_BLOCK_SIZE);let mut values =encoder.allocate_scratch(size_for_shape(&[batch_dim, self.input_dimension], DataType::I8))?;let mut scales = encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::F32))?;- let mut group_sums = preparation- .emit_group_sums+ let emit_group_sums = quantize_transform.emit_group_sums();+ let mut group_sums = emit_group_sums.then(|| encoder.allocate_scratch(size_for_shape(&[batch_dim, groups_per_row], DataType::I32))).transpose()?;- preparation.kernel.encode(+ quantize_transform.encode_quantize(&input,&mut values,&mut scales,@@ -203,7 +198,6 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {&self.input_factors,batch_dim as u32,self.input_dimension as u32,- HADAMARD_TRANSFORM_BLOCK_SIZE as u32,encoder,);return self.inner_linear.encode_with_a(@@ -218,11 +212,11 @@ impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {}let mut input = input;- self.input_hadamard_kernel.encode(+ self.input_transform.encode_fp_in_place(&mut input,&self.input_factors,- self.input_dimension as u32,batch_dim as u32,+ self.input_dimension as u32,encoder,);self.inner_linear.encode(input, batch_dim, encoder)diff --git a/crates/backend-uzu/src/tests/matmul/mod.rs b/crates/backend-uzu/src/tests/matmul/mod.rsindex 754732966..18c9c7a53 100644--- a/crates/backend-uzu/src/tests/matmul/mod.rs+++ b/crates/backend-uzu/src/tests/matmul/mod.rs@@ -9,7 +9,7 @@ pub use harness::run_metal;pub use harness::{Case, cpu_reference, deterministic_input};#[cfg(backend = "metal")]pub use quant::run_quant_metal;-pub use quant::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu};+pub use quant::{QuantBuffers, QuantInput, quant_arguments, quant_b_variant, run_quant_cpu};pub use shape::{Shape, all_correctness_shapes, bench_fp_gemm_shapes, bench_quant_gemm_shapes, bench_quant_gemv_shapes,qwen3_layer_shapes,diff --git a/crates/backend-uzu/src/tests/matmul/quant.rs b/crates/backend-uzu/src/tests/matmul/quant.rsindex 6675f7984..aa7c96af4 100644--- a/crates/backend-uzu/src/tests/matmul/quant.rs+++ b/crates/backend-uzu/src/tests/matmul/quant.rs@@ -18,7 +18,7 @@ use crate::{},cpu::{Cpu,- kernel::rht_quantize_activations::{min_max_symmetric_divisor, quantize_symmetric_i8},+ kernel::activation_transform::{min_max_symmetric_divisor, quantize_symmetric_i8},},},tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},@@ -42,6 +42,7 @@ pub struct QuantInput<T: ArrayElement + Float> {pub group_size: u32,pub quant_method: QuantizationMethod,pub mode: QuantizationMode,+ pub signed_codes: bool,pub prepared_a: Option<PreparedInt8A>,}@@ -99,11 +100,18 @@ impl<T: ArrayElement + Float> QuantInput<T> {group_size,quant_method,mode: mode_for_bits(bits),+ signed_codes: false,prepared_a: None,}}+ pub fn with_signed_weight_codes(mut self) -> Self {+ self.signed_codes = true;+ self+ }+pub fn with_prepared_a(mut self) -> Self {+ self.signed_codes = true;let group_size = HADAMARD_TRANSFORM_BLOCK_SIZE;let rows = self.m as usize;let columns = self.k as usize;@@ -138,8 +146,14 @@ impl<T: ArrayElement + Float> QuantInput<T> {self}- fn weights_for_upload(&self) -> Vec<u32> {- self.w_packed.clone()+ pub(crate) fn weights_for_upload(&self) -> Vec<u32> {+ let mut words = self.w_packed.clone();+ let sign_flip_mask = self.signed_codes.then(|| self.mode.weight_codes_sign_flip_mask()).flatten();+ if let Some(mask) = sign_flip_mask {+ let broadcast_mask = u32::from(mask) * 0x0101_0101;+ words.iter_mut().for_each(|word| *word ^= broadcast_mask);+ }+ words}pub(crate) fn weight_buffer_bytes(&self) -> usize {@@ -192,51 +206,77 @@ impl<B: Backend, T: ArrayElement + Float> QuantBuffers<B, T> {}}-pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(- buffers: &'a mut QuantBuffers<B, T>,+pub fn quant_b_variant<'a, B: Backend, T: ArrayElement + Float>(+ w: &'a Allocation<B>,+ scales: &'a Allocation<B>,+ zero_points: Option<&'a Allocation<B>>,+ biases: Option<&'a Allocation<B>>,input: &QuantInput<T>,-) -> MatmulArguments<'a, 'a, 'a, B> {- let b_variant = match input.quant_method {+) -> MatmulB<'a, B> {+ let signed_codes = input.signed_codes;+ match input.quant_method {QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {- b: &buffers.w,- scales: &buffers.scales,- biases: buffers.bias.as_ref().expect("bias buffer"),+ b: w,+ scales,+ biases: biases.expect("bias buffer"),mode: input.mode,group_size: input.group_size,+ signed_codes,},QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {- b: &buffers.w,- scales: &buffers.scales,- zero_points: buffers.zp.as_ref().expect("zp buffer"),+ b: w,+ scales,+ zero_points: zero_points.expect("zp buffer"),mode: input.mode,group_size: input.group_size,+ signed_codes,},QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {- b: &buffers.w,- scales: &buffers.scales,+ b: w,+ scales,mode: input.mode,group_size: input.group_size,+ signed_codes,},- };+ }+}++pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(+ buffers: &'a mut QuantBuffers<B, T>,+ input: &QuantInput<T>,+) -> MatmulArguments<'a, 'a, 'a, B> {+ let QuantBuffers {+ w,+ scales,+ zp,+ bias,+ x,+ prepared_a,+ prepared_a_scales,+ prepared_a_group_sums,+ y,+ ..+ } = buffers;+ let b = quant_b_variant(w, scales, zp.as_ref(), bias.as_ref(), input);let a = match &input.prepared_a {Some(_) => MatmulA::Int8Symmetric {- values: buffers.prepared_a.as_ref().expect("prepared activation buffer"),- scales: buffers.prepared_a_scales.as_ref().expect("prepared activation scales"),+ values: prepared_a.as_ref().expect("prepared activation buffer"),+ scales: prepared_a_scales.as_ref().expect("prepared activation scales"),// Symmetric weights carry no correction term, so the GEMM never reads these.group_sums: (input.quant_method != QuantizationMethod::ScaleSymmetric)- .then(|| buffers.prepared_a_group_sums.as_ref().expect("prepared activation row sums")),+ .then(|| prepared_a_group_sums.as_ref().expect("prepared activation row sums")),},None => MatmulA::FullPrecision {- values: &buffers.x,+ values: x,offset: 0,},};MatmulArguments {a,- b: b_variant,+ b,b_leading_dimension: None,b_transpose: true,- d: &mut buffers.y,+ d: y,d_transform: MatmulDOps::none(),gather_indices: None,m: input.m,diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rsnew file mode 100644index 000000000..edc33256b--- /dev/null+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/activation_transform_test.rs@@ -0,0 +1,260 @@+use std::fmt::Debug;++use half::bf16;+use num_traits::Float;+use proc_macros::uzu_test;++use crate::{+ array::ArrayElement,+ backends::common::{+ Backend, Context, Encoder, gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE as BLOCK_SIZE, kernel::ActivationTransform,+ },+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_backend},+};++#[derive(Clone, Copy, Debug)]+enum TransformOrder {+ Input,+ Output,+}++fn reference_transform(+ data: &[f64],+ factors: &[i32],+ channel_count: usize,+ order: TransformOrder,+) -> Vec<f64> {+ let batch_count = data.len() / channel_count;+ let normalization_factor = 1.0 / (BLOCK_SIZE as f64).sqrt();+ let mut result = data.to_vec();++ for batch_index in 0..batch_count {+ let batch_offset = batch_index * channel_count;+ for block_start in (0..channel_count).step_by(BLOCK_SIZE) {+ if matches!(order, TransformOrder::Input) {+ for lane in 0..BLOCK_SIZE {+ let index = batch_offset + block_start + lane;+ result[index] *= f64::from(factors[block_start + lane]);+ }+ }++ let mut stride = 1;+ while stride < BLOCK_SIZE {+ for pair_start in (0..BLOCK_SIZE).step_by(stride * 2) {+ for offset in 0..stride {+ let index_a = batch_offset + block_start + pair_start + offset;+ let index_b = index_a + stride;+ let sum = result[index_a] + result[index_b];+ let difference = result[index_a] - result[index_b];+ result[index_a] = sum;+ result[index_b] = difference;+ }+ }+ stride *= 2;+ }++ for lane in 0..BLOCK_SIZE {+ let index = batch_offset + block_start + lane;+ result[index] *= normalization_factor;+ if matches!(order, TransformOrder::Output) {+ result[index] *= f64::from(factors[block_start + lane]);+ }+ }+ }+ }++ result+}++fn run<T: ArrayElement + Float, B: Backend>(+ data: &[T],+ factors: &[i32],+ channel_count: usize,+ order: TransformOrder,+ in_place: bool,+) -> Vec<T> {+ let context = B::Context::new().expect("context");+ let kernel = match order {+ TransformOrder::Input => ActivationTransform::<B>::input_rht(context.as_ref(), T::data_type(), in_place),+ TransformOrder::Output => ActivationTransform::<B>::output_rht(context.as_ref(), T::data_type(), in_place),+ }+ .expect("activation transform");++ let mut input = alloc_allocation_with_data::<B, T>(context.as_ref(), data);+ let mut output = alloc_allocation::<B, T>(context.as_ref(), data.len());+ let factors = alloc_allocation_with_data::<B, i32>(context.as_ref(), factors);+ let batch_count = (data.len() / channel_count) as u32;+ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");+ if in_place {+ kernel.encode_fp_in_place(&mut input, &factors, batch_count, channel_count as u32, &mut encoder);+ } else {+ kernel.encode_fp(&input, &mut output, &factors, batch_count, channel_count as u32, &mut encoder);+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec(if in_place {+ &input+ } else {+ &output+ })+}++fn check<T: ArrayElement + Float + Debug>(tolerance: f64) {+ for (order, in_place) in [+ (TransformOrder::Input, false),+ (TransformOrder::Input, true),+ (TransformOrder::Output, false),+ (TransformOrder::Output, true),+ ] {+ for (batch_count, channel_count) in [(1, 32), (1, 64), (1, 128), (4, 32), (4, 256), (2, 2048)] {+ let data_f64: Vec<f64> =+ (0..batch_count * channel_count).map(|index| ((index as f64) * 0.1).sin() * 2.0).collect();+ let factors: Vec<i32> = (0..channel_count)+ .map(|index| {+ if index % 3 == 0 {+ -1+ } else {+ 1+ }+ })+ .collect();+ let expected = reference_transform(&data_f64, &factors, channel_count, order);+ let data: Vec<T> = data_f64.iter().map(|&value| T::from(value).unwrap()).collect();++ for_each_backend!(|B| {+ let actual = run::<T, B>(&data, &factors, channel_count, order, in_place);+ for (index, (actual_value, &expected_value)) in actual.iter().zip(&expected).enumerate() {+ let actual_value = actual_value.to_f64().unwrap();+ let error = (actual_value - expected_value).abs();+ assert!(+ error <= (expected_value.abs() * tolerance).max(tolerance),+ "{order:?} (in_place={in_place}) mismatch at {index} for batch={batch_count}, \+ channels={channel_count}: actual={actual_value}, expected={expected_value}, error={error}"+ );+ }+ });+ }+ }+}++#[uzu_test]+fn input_and_output_rht_f32() {+ check::<f32>(1e-4);+}++#[uzu_test]+fn input_and_output_rht_bf16() {+ check::<bf16>(0.1);+}++#[cfg(backend = "metal")]+mod quantize {+ use proc_macros::uzu_test;+ use rand::{RngExt, SeedableRng, rngs::SmallRng};++ use super::BLOCK_SIZE;+ use crate::{+ backends::{+ common::{Backend, Context, Encoder, kernel::ActivationTransform},+ cpu::Cpu,+ metal::{Metal, MetalContext},+ },+ data_type::DataType,+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ };++ fn check_quantize(emit_group_sums: bool) {+ let rows = 5usize;+ let columns = 96usize;+ let groups = columns / BLOCK_SIZE;++ let mut rng = SmallRng::seed_from_u64(0x5EED_0001);+ let input_data: Vec<f32> = (0..rows * columns).map(|_| rng.random_range(-1.0f32..1.0f32)).collect();+ let factors_data: Vec<i32> = (0..columns)+ .map(|i| {+ if i % 3 == 0 {+ -1+ } else {+ 1+ }+ })+ .collect();++ let metal = MetalContext::new().expect("metal");+ let cpu = <Cpu as Backend>::Context::new().expect("cpu");+ let mut metal_values = alloc_allocation::<Metal, i8>(metal.as_ref(), rows * columns);+ let mut metal_scales = alloc_allocation::<Metal, f32>(metal.as_ref(), rows * groups);+ let mut cpu_values = alloc_allocation::<Cpu, i8>(cpu.as_ref(), rows * columns);+ let mut cpu_scales = alloc_allocation::<Cpu, f32>(cpu.as_ref(), rows * groups);+ let mut metal_group_sums =+ emit_group_sums.then(|| alloc_allocation::<Metal, i32>(metal.as_ref(), rows * groups));+ let mut cpu_group_sums = emit_group_sums.then(|| alloc_allocation::<Cpu, i32>(cpu.as_ref(), rows * groups));+ let metal_input = alloc_allocation_with_data::<Metal, f32>(metal.as_ref(), &input_data);+ let metal_factors = alloc_allocation_with_data::<Metal, i32>(metal.as_ref(), &factors_data);+ let cpu_input = alloc_allocation_with_data::<Cpu, f32>(cpu.as_ref(), &input_data);+ let cpu_factors = alloc_allocation_with_data::<Cpu, i32>(cpu.as_ref(), &factors_data);++ let metal_kernel =+ ActivationTransform::quantize(metal.as_ref(), DataType::F32, emit_group_sums).expect("metal prepare");+ let cpu_kernel =+ ActivationTransform::quantize(cpu.as_ref(), DataType::F32, emit_group_sums).expect("cpu prepare");++ let mut metal_enc = Encoder::<Metal>::new(metal.as_ref()).expect("metal encoder");+ metal_kernel.encode_quantize(+ &metal_input,+ &mut metal_values,+ &mut metal_scales,+ metal_group_sums.as_mut(),+ &metal_factors,+ rows as u32,+ columns as u32,+ &mut metal_enc,+ );+ metal_enc.end_encoding().submit().wait_until_completed().unwrap();++ let mut cpu_enc = Encoder::<Cpu>::new(cpu.as_ref()).expect("cpu encoder");+ cpu_kernel.encode_quantize(+ &cpu_input,+ &mut cpu_values,+ &mut cpu_scales,+ cpu_group_sums.as_mut(),+ &cpu_factors,+ rows as u32,+ columns as u32,+ &mut cpu_enc,+ );+ cpu_enc.end_encoding().submit().wait_until_completed().unwrap();++ let (mv, ms) = (allocation_to_vec::<Metal, i8>(&metal_values), allocation_to_vec::<Metal, f32>(&metal_scales));+ let (cv, cs) = (allocation_to_vec::<Cpu, i8>(&cpu_values), allocation_to_vec::<Cpu, f32>(&cpu_scales));+ for (i, (a, e)) in ms.iter().zip(&cs).enumerate() {+ let rel = (a - e).abs() / e.abs().max(1e-6);+ assert!(rel < 1e-3, "scale {i}: {a} != {e}");+ }+ assert!(mv.iter().zip(&cv).all(|(a, e)| (*a as i32 - *e as i32).abs() <= 1));++ if let (Some(metal_group_sums), Some(cpu_group_sums)) = (&metal_group_sums, &cpu_group_sums) {+ let mrs = allocation_to_vec::<Metal, i32>(metal_group_sums);+ let crs = allocation_to_vec::<Cpu, i32>(cpu_group_sums);+ for (codes, sums, label) in [(&mv, &mrs, "metal"), (&cv, &crs, "cpu")] {+ for row in 0..rows {+ for group in 0..groups {+ let start = row * columns + group * BLOCK_SIZE;+ let expected: i32 = codes[start..start + BLOCK_SIZE].iter().map(|code| *code as i32).sum();+ assert_eq!(sums[row * groups + group], expected, "{label} group_sum r{row} g{group}");+ }+ }+ }+ assert!(mrs.iter().zip(&crs).all(|(a, e)| (a - e).abs() <= BLOCK_SIZE as i32));+ }+ }++ #[uzu_test]+ fn quantize_with_group_sums_matches_cpu() {+ check_quantize(true);+ }++ #[uzu_test]+ fn quantize_without_group_sums_matches_cpu() {+ check_quantize(false);+ }+}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/hadamard_transform_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/hadamard_transform_test.rsdeleted file mode 100644index 3dbb9c268..000000000--- a/crates/backend-uzu/tests/unit/backends/common/kernel/hadamard_transform_test.rs+++ /dev/null@@ -1,154 +0,0 @@-use std::fmt::Debug;--use half::bf16;-use num_traits::Float;-use proc_macros::uzu_test;--use crate::{- array::ArrayElement,- backends::common::{- Backend, Context, Encoder, Kernels, gpu_types::HadamardTransformOrder, kernel::HadamardTransformKernel,- },- tests::helpers::{alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},-};--const BLOCK_SIZE: usize = 32;--fn reference_hadamard_transform_mul(- data: &[f64],- factors: &[f64],- channel_count: usize,-) -> Vec<f64> {- let batch_count = data.len() / channel_count;- let normalization_factor = 1.0 / (BLOCK_SIZE as f64).sqrt();- let mut result = data.to_vec();-- for batch_index in 0..batch_count {- let batch_offset = batch_index * channel_count;-- for block_start in (0..channel_count).step_by(BLOCK_SIZE) {- for lane in 0..BLOCK_SIZE {- let index = batch_offset + block_start + lane;- let factor_index = block_start + lane;- result[index] *= factors[factor_index];- }-- let mut stride = 1;- while stride < BLOCK_SIZE {- for pair_start in (0..BLOCK_SIZE).step_by(stride * 2) {- for offset in 0..stride {- let index_a = batch_offset + block_start + pair_start + offset;- let index_b = index_a + stride;- let sum = result[index_a] + result[index_b];- let difference = result[index_a] - result[index_b];- result[index_a] = sum;- result[index_b] = difference;- }- }- stride *= 2;- }-- for lane in 0..BLOCK_SIZE {- let index = batch_offset + block_start + lane;- result[index] *= normalization_factor;- }- }- }-- result-}--struct TestInput<T: ArrayElement + Float> {- data: Box<[T]>,- factors: Box<[i32]>,- channel_count: usize,- batch_count: usize,-}--fn generate_test_input<T: ArrayElement + Float>(- batch_count: usize,- channel_count: usize,-) -> (TestInput<T>, Vec<f64>) {- let total_elements = batch_count * channel_count;-- let data_f64: Vec<f64> = (0..total_elements).map(|index| ((index as f64) * 0.1).sin() * 2.0).collect();-- let factors_i32: Vec<i32> = (0..channel_count)- .map(|index| {- if index % 3 == 0 {- -1- } else {- 1- }- })- .collect();-- let factors_f64: Vec<f64> = factors_i32.iter().map(|&v| v as f64).collect();- let expected = reference_hadamard_transform_mul(&data_f64, &factors_f64, channel_count);-- let data: Vec<T> = data_f64.iter().map(|&value| T::from(value).unwrap()).collect();-- let input = TestInput {- data: data.into_boxed_slice(),- factors: factors_i32.into_boxed_slice(),- channel_count,- batch_count,- };-- (input, expected)-}--fn run_kernel<T: ArrayElement + Float, B: Backend>(input: &TestInput<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create context");-- let kernel = <<B as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(- &context,- T::data_type(),- HadamardTransformOrder::Input,- )- .expect("Failed to create HadamardTransformKernel");-- let mut data = alloc_allocation_with_data::<B, T>(&context, &input.data);- let factors_allocation = alloc_allocation_with_data::<B, i32>(&context, &input.factors);-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(&mut data, &factors_allocation, input.channel_count as u32, input.batch_count as u32, &mut encoder);- encoder.end_encoding().submit().wait_until_completed().unwrap();-- allocation_to_vec(&data)-}--fn test_hadamard_transform<T: ArrayElement + Float + Debug>(tolerance: f64) {- let test_cases = [(1, 32), (1, 64), (1, 128), (4, 32), (4, 256), (2, 2048)];-- for (batch_count, channel_count) in test_cases {- let (input, expected) = generate_test_input::<T>(batch_count, channel_count);-- for_each_non_cpu_backend!(|B| {- let actual = run_kernel::<T, B>(&input);- assert_eq!(actual.len(), expected.len());-- for (index, (actual_value, &expected_value)) in actual.iter().zip(expected.iter()).enumerate() {- let actual_f64 = actual_value.to_f64().unwrap();- let error = (actual_f64 - expected_value).abs();- let relative_bound = expected_value.abs() * tolerance;- let absolute_bound = tolerance;- assert!(- error <= relative_bound.max(absolute_bound),- "Mismatch at index {index} for batch_count={batch_count}, channel_count={channel_count}: \- actual={actual_f64}, expected={expected_value}, error={error}"- );- }- });- }-}--#[uzu_test]-fn test_f32() {- test_hadamard_transform::<f32>(1e-4);-}--#[uzu_test]-fn test_bf16() {- test_hadamard_transform::<bf16>(0.1);-}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rsindex 38b4ad6cd..b5e209ec4 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/a8w_bench.rs@@ -10,10 +10,10 @@ use crate::{backends::{common::{Allocation, Backend, Encoder,- gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, HadamardTransformOrder, QuantizationMethod, QuantizationMode},+ gpu_types::{HADAMARD_TRANSFORM_BLOCK_SIZE, QuantizationMethod, QuantizationMode},kernel::{- HadamardTransformKernel, Kernels, RHTQuantizeActivationsKernel,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ ActivationTransform, Kernels,+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulShape},},},metal::{DeviceTier, GemmDispatchPath, GemvDispatch, GemvSpecialization, Metal, MetalContext},@@ -27,8 +27,6 @@ use crate::{};type MetalMatmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel;-type MetalPrepare = <<Metal as Backend>::Kernels as Kernels>::RHTQuantizeActivationsKernel;-type MetalHadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel;#[derive(Clone, Copy)]enum BenchPath {@@ -48,7 +46,8 @@ impl BenchPath {}struct BenchmarkData {- weights_u8: Allocation<Metal>,+ unsigned_weights: Allocation<Metal>,+ signed_weights: Allocation<Metal>,weight_scales: Allocation<Metal>,activations: Allocation<Metal>,rht_factors: Allocation<Metal>,@@ -72,9 +71,11 @@ impl BenchmarkData {seed: u64,) -> Self {let group_size = HADAMARD_TRANSFORM_BLOCK_SIZE as u32;- let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, QuantizationMethod::ScaleSymmetric, seed);+ let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, QuantizationMethod::ScaleSymmetric, seed)+ .with_prepared_a();- let weights_u8 = alloc_allocation_with_data::<Metal, u32>(context, &input.w_packed);+ let unsigned_weights = alloc_allocation_with_data::<Metal, u32>(context, &input.w_packed);+ let signed_weights = alloc_allocation_with_data::<Metal, u32>(context, &input.weights_for_upload());let weight_scales = alloc_allocation_with_data::<Metal, bf16>(context, &input.scales);let activations = alloc_allocation_with_data::<Metal, bf16>(context, &input.x);let rht: Vec<i32> = (0..k)@@ -90,7 +91,8 @@ impl BenchmarkData {let groups = k / group_size as usize;Self {- weights_u8,+ unsigned_weights,+ signed_weights,weight_scales,activations,rht_factors,@@ -119,10 +121,11 @@ impl BenchmarkData {offset: 0,},b: MatmulB::ScaleSymmetricDequant {- b: &self.weights_u8,+ b: &self.unsigned_weights,scales: &self.weight_scales,mode: self.mode,group_size: self.group_size,+ signed_codes: false,},b_leading_dimension: None,b_transpose: true,@@ -141,8 +144,8 @@ fn encode_step(path: BenchPath,data: &mut BenchmarkData,output: &mut Allocation<Metal>,- prepare: &MetalPrepare,- hadamard: &MetalHadamard,+ prepare: &ActivationTransform<Metal>,+ hadamard: &ActivationTransform<Metal>,matmul: &mut MetalMatmul,gemv: &mut GemvDispatch,device_tier: DeviceTier,@@ -150,7 +153,7 @@ fn encode_step() {match path {BenchPath::A8GemmMxu => {- prepare.encode(+ prepare.encode_quantize(&data.activations,&mut data.a_int8,&mut data.a_scales,@@ -158,7 +161,6 @@ fn encode_step(&data.rht_factors,data.m,data.k,- HADAMARD_TRANSFORM_BLOCK_SIZE as u32,encoder,);let args: MatmulArguments<'_, '_, '_, Metal, &Allocation<Metal>> = MatmulArguments {@@ -168,10 +170,11 @@ fn encode_step(group_sums: None,},b: MatmulB::ScaleSymmetricDequant {- b: &data.weights_u8,+ b: &data.signed_weights,scales: &data.weight_scales,mode: data.mode,group_size: data.group_size,+ signed_codes: true,},b_leading_dimension: None,b_transpose: true,@@ -186,16 +189,22 @@ fn encode_step(},BenchPath::Bf16GemmMxu => {encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);- hadamard.encode(&mut data.a_working, &data.rht_factors, data.k, data.m, encoder);+ hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);let args = data.bf16_arguments(output);matmul.gemm.encode_dispatch_path(args, GemmDispatchPath::Mxu, encoder).expect("bf16 gemm mxu encode");},BenchPath::Bf16Gemv => {encoder.encode_copy(&data.activations, .., &mut data.a_working, ..);- hadamard.encode(&mut data.a_working, &data.rht_factors, data.k, data.m, encoder);+ hadamard.encode_fp_in_place(&mut data.a_working, &data.rht_factors, data.m, data.k, encoder);let args = data.bf16_arguments(output);- let spec = GemvSpecialization::select(&args, DataType::BF16, DataType::BF16, DataType::BF16, device_tier)- .expect("bf16 gemv specialization");+ let spec = GemvSpecialization::select_shape(+ &MatmulShape::from_arguments(&args),+ DataType::BF16,+ DataType::BF16,+ DataType::BF16,+ device_tier,+ )+ .expect("bf16 gemv specialization");gemv.encode(args, spec, encoder).expect("bf16 gemv encode");},}@@ -205,8 +214,8 @@ fn bench_bits(c: &mut Criterion,context: &MetalContext,device_tier: DeviceTier,- prepare: &MetalPrepare,- hadamard: &MetalHadamard,+ prepare: &ActivationTransform<Metal>,+ hadamard: &ActivationTransform<Metal>,bits: u32,) {let mut matmul = <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16)@@ -224,8 +233,8 @@ fn bench_bits(let mut output = alloc_allocation::<Metal, bf16>(context, m * n);let shape_label = format!("{layer}_m{m}_k{k}_n{n}");- let gemv_eligible = GemvSpecialization::select(- &data.bf16_arguments(&mut output),+ let gemv_eligible = GemvSpecialization::select_shape(+ &MatmulShape::from_arguments(&data.bf16_arguments(&mut output)),DataType::BF16,DataType::BF16,DataType::BF16,@@ -270,11 +279,8 @@ fn bench_a8w(c: &mut Criterion) {}let device_tier = context.device_tier();- let prepare =- <MetalPrepare as RHTQuantizeActivationsKernel>::new(&context, DataType::BF16, false).expect("prepare kernel");- let hadamard =- <MetalHadamard as HadamardTransformKernel>::new(&context, DataType::BF16, HadamardTransformOrder::Input)- .expect("hadamard kernel");+ let prepare = ActivationTransform::<Metal>::quantize(&context, DataType::BF16, false).expect("prepare kernel");+ let hadamard = ActivationTransform::<Metal>::input_rht(&context, DataType::BF16, true).expect("hadamard kernel");for bits in [8u32, 4u32] {bench_bits(c, &context, device_tier, &prepare, &hadamard, bits);diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rsindex acdc15813..c4fb8a95f 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/gemv_test.rs@@ -21,7 +21,7 @@ use crate::{tests::{assert::assert_eq_float,helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, for_each_non_cpu_backend},- matmul::{QuantBuffers, QuantInput},+ matmul::{QuantBuffers, QuantInput, quant_b_variant},},};@@ -249,28 +249,8 @@ fn gemv_gather() {let context = <B as Backend>::Context::new().expect("context");let buffers = QuantBuffers::<B, bf16>::allocate(&context, &input);let ids_alloc = alloc_allocation_with_data::<B, u32>(&context, &ids);- let variant = || match method {- QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {- b: &buffers.w,- scales: &buffers.scales,- biases: buffers.bias.as_ref().expect("bias buffer"),- mode: input.mode,- group_size: input.group_size,- },- QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {- b: &buffers.w,- scales: &buffers.scales,- zero_points: buffers.zp.as_ref().expect("zp buffer"),- mode: input.mode,- group_size: input.group_size,- },- QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {- b: &buffers.w,- scales: &buffers.scales,- mode: input.mode,- group_size: input.group_size,- },- };+ let variant =+ || quant_b_variant(&buffers.w, &buffers.scales, buffers.zp.as_ref(), buffers.bias.as_ref(), &input);(run_gemv::<B, bf16>(&context, &buffers.x, variant(), None, m, vocab, k, None),run_gemv::<B, bf16>(&context, &buffers.x, variant(), Some(&ids_alloc), m, ids_per_row, k, None),diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rsindex a4065ed74..bcce9d754 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs@@ -26,7 +26,7 @@ use crate::{},tests::{helpers::allocation_to_vec,- matmul::{QuantBuffers, QuantInput, quant_arguments, run_quant_cpu, run_quant_metal},+ matmul::{QuantBuffers, QuantInput, quant_arguments, quant_b_variant, run_quant_cpu, run_quant_metal},},};@@ -524,6 +524,38 @@ fn a8w_mxu_parity_bf16();}+#[rstest]+#[test_attr(uzu_test)]+#[case::w4_sym(4u32, QuantizationMethod::ScaleSymmetric, 256usize)]+#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256usize)]+#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256usize)]+#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256usize)]+#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256usize)]+#[case::w8_zp(8u32, QuantizationMethod::ScaleZeroPoint, 256usize)]+#[case::w8_zp_tail(8u32, QuantizationMethod::ScaleZeroPoint, 224usize)]+fn signed_weights_full_precision_activations_parity_bf16(+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+ #[case] k: usize,+) {+ let context = MetalContext::new().expect("Metal context");+ let (m, n, group_size) = (2usize, 128usize, 32u32);+ let input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0).with_signed_weight_codes();+ let reference_input = QuantInput::<bf16>::new(m, k, n, group_size, bits, method, 0);+ let reference = run_quant_cpu::<bf16>(&reference_input);++ for (label, path) in [("gemv", None), ("simdgroup", Some(GemmDispatchPath::Simdgroup))] {+ let actual = run_quant_metal::<bf16>(&context, &input, path);+ assert_parity::<bf16>(+ &format!("signed weights FP-A {label} bits={bits} method={method:?}"),+ &reference,+ &actual,+ 0.05,+ 0.5,+ );+ }+}+#[rstest]#[test_attr(uzu_test)]#[case::w4_bias(4u32, false)]@@ -628,38 +660,13 @@ fn run_widened_f32<B: Backend>(context: &B::Context,input: &QuantInput<bf16>,) -> Vec<f32> {- use crate::{- backends::common::kernel::matmul::{MatmulA, MatmulB},- data_type::DataType,- tests::helpers::alloc_allocation,- };+ use crate::{backends::common::kernel::matmul::MatmulA, data_type::DataType, tests::helpers::alloc_allocation};let buffers = QuantBuffers::<B, bf16>::allocate(context, input);let mut y = alloc_allocation::<B, f32>(context, (input.m as usize) * (input.n as usize));let mut matmul =<<B as Backend>::Kernels as Kernels>::MatmulKernel::new(context, DataType::BF16, DataType::BF16, DataType::F32).expect("MatmulKernel widened");- let b: MatmulB<'_, B> = match input.quant_method {- QuantizationMethod::ScaleBias => MatmulB::ScaleBiasDequant {- b: &buffers.w,- scales: &buffers.scales,- biases: buffers.bias.as_ref().expect("bias buffer"),- mode: input.mode,- group_size: input.group_size,- },- QuantizationMethod::ScaleZeroPoint => MatmulB::ScaleZeroPointDequant {- b: &buffers.w,- scales: &buffers.scales,- zero_points: buffers.zp.as_ref().expect("zp buffer"),- mode: input.mode,- group_size: input.group_size,- },- QuantizationMethod::ScaleSymmetric => MatmulB::ScaleSymmetricDequant {- b: &buffers.w,- scales: &buffers.scales,- mode: input.mode,- group_size: input.group_size,- },- };+ let b = quant_b_variant(&buffers.w, &buffers.scales, buffers.zp.as_ref(), buffers.bias.as_ref(), input);let mut encoder = Encoder::<B>::new(context).expect("encoder");matmul.encode(diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rsindex b4cfbf441..8e75c785b 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs@@ -1,9 +1,9 @@mod activation_test;+mod activation_transform_test;mod attention;mod embedding;mod gated_act_mul_test;mod gdn;-mod hadamard_transform_test;mod kv_cache_update_test;mod logit_soft_cap_test;mod matmul;diff --git a/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs b/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rsdeleted file mode 100644index 91f3d7251..000000000--- a/crates/backend-uzu/tests/unit/backends/metal/kernel/rht_quantize_activations_test.rs+++ /dev/null@@ -1,106 +0,0 @@-#![cfg(backend = "metal")]--use proc_macros::uzu_test;-use rand::{RngExt, SeedableRng, rngs::SmallRng};--use super::RHTQuantizeActivationsMetalKernel;-use crate::{- backends::{- common::{- Backend, Context, Encoder,- kernel::{Kernels, RHTQuantizeActivationsKernel},- },- cpu::Cpu,- metal::{Metal, MetalContext},- },- data_type::DataType,- tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},-};--#[uzu_test]-fn rht_quantize_matches_cpu() {- let rows = 5usize;- let columns = 96usize;- let group_size = 32u32;- let groups = columns.div_ceil(group_size as usize);-- let mut rng = SmallRng::seed_from_u64(0x5EED_0001);- let input_data: Vec<f32> = (0..rows * columns).map(|_| rng.random_range(-1.0f32..1.0f32)).collect();- let factors_data: Vec<i32> = (0..columns)- .map(|i| {- if i % 3 == 0 {- -1- } else {- 1- }- })- .collect();-- let metal = MetalContext::new().expect("metal");- let cpu = <Cpu as Backend>::Context::new().expect("cpu");- let mut metal_values = alloc_allocation::<Metal, i8>(&metal, rows * columns);- let mut metal_scales = alloc_allocation::<Metal, f32>(&metal, rows * groups);- let mut cpu_values = alloc_allocation::<Cpu, i8>(&cpu, rows * columns);- let mut cpu_scales = alloc_allocation::<Cpu, f32>(&cpu, rows * groups);- let mut metal_group_sums = alloc_allocation::<Metal, i32>(&metal, rows * groups);- let mut cpu_group_sums = alloc_allocation::<Cpu, i32>(&cpu, rows * groups);-- let metal_input = alloc_allocation_with_data::<Metal, f32>(&metal, &input_data);- let metal_factors = alloc_allocation_with_data::<Metal, i32>(&metal, &factors_data);- let cpu_input = alloc_allocation_with_data::<Cpu, f32>(&cpu, &input_data);- let cpu_factors = alloc_allocation_with_data::<Cpu, i32>(&cpu, &factors_data);-- let metal_kernel = RHTQuantizeActivationsMetalKernel::new(&metal, DataType::F32, true).expect("metal prepare");- let cpu_kernel =- <<Cpu as Backend>::Kernels as Kernels>::RHTQuantizeActivationsKernel::new(&cpu, DataType::F32, true)- .expect("cpu prepare");-- let mut metal_enc = Encoder::<Metal>::new(&metal).expect("metal encoder");- metal_kernel.encode(- &metal_input,- &mut metal_values,- &mut metal_scales,- Some(&mut metal_group_sums),- &metal_factors,- rows as u32,- columns as u32,- group_size,- &mut metal_enc,- );- metal_enc.end_encoding().submit().wait_until_completed().unwrap();-- let mut cpu_enc = Encoder::<Cpu>::new(&cpu).expect("cpu encoder");- cpu_kernel.encode(- &cpu_input,- &mut cpu_values,- &mut cpu_scales,- Some(&mut cpu_group_sums),- &cpu_factors,- rows as u32,- columns as u32,- group_size,- &mut cpu_enc,- );- cpu_enc.end_encoding().submit().wait_until_completed().unwrap();-- let (mv, ms) = (allocation_to_vec::<Metal, i8>(&metal_values), allocation_to_vec::<Metal, f32>(&metal_scales));- let (cv, cs) = (allocation_to_vec::<Cpu, i8>(&cpu_values), allocation_to_vec::<Cpu, f32>(&cpu_scales));- for (i, (a, e)) in ms.iter().zip(&cs).enumerate() {- let rel = (a - e).abs() / e.abs().max(1e-6);- assert!(rel < 1e-3, "scale {i}: {a} != {e}");- }- assert!(mv.iter().zip(&cv).all(|(a, e)| (*a as i32 - *e as i32).abs() <= 1));-- let mrs = allocation_to_vec::<Metal, i32>(&metal_group_sums);- let crs = allocation_to_vec::<Cpu, i32>(&cpu_group_sums);- for (codes, sums, label) in [(&mv, &mrs, "metal"), (&cv, &crs, "cpu")] {- for row in 0..rows {- for group in 0..groups {- let start = row * columns + group * group_size as usize;- let expected: i32 = codes[start..start + group_size as usize].iter().map(|code| *code as i32).sum();- assert_eq!(sums[row * groups + group], expected, "{label} group_sum r{row} g{group}");- }- }- }- assert!(mrs.iter().zip(&crs).all(|(a, e)| (a - e).abs() <= group_size as i32));-}