A high-performance inference engine for AI models
Trellis projection for M5 (#888)
CCYeh663d27ffe9c9
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rsindex 305dad425..3f8897fa3 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rs@@ -1,4 +1,4 @@-use super::{d_ops::MatmulDOps, matmul_a::MatmulA, matmul_b::MatmulB};+use super::{MatmulOutput, matmul_a::MatmulA, matmul_b::MatmulB};use crate::backends::common::{Backend, BufferMut, BufferRef};pub struct MatmulArguments<@@ -13,8 +13,7 @@ pub struct MatmulArguments<pub b: MatmulB<TB>,pub b_leading_dimension: Option<u32>,pub b_transpose: bool,- pub d: TD,- pub d_transform: MatmulDOps<'d, B>,+ pub output: MatmulOutput<'d, B, TD>,pub gather_indices: Option<TI>,pub m: u32,pub n: u32,diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rsindex 808aaf441..24d59b226 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs@@ -17,10 +17,16 @@ pub enum MatmulError<B: Backend> {bit: GemmDTransform,path: &'static str,},- #[error("Unsupported B layout on path {path}")]+ #[error("Unsupported layout on path {path}")]UnsupportedLayout {path: &'static str,},+ #[error("Unsupported output row stride {stride} for {columns} columns on path {path}")]+ UnsupportedOutputStride {+ path: &'static str,+ stride: u32,+ columns: u32,+ },#[error("Incompatible A operand for {path}: {reason}")]IncompatibleA {path: &'static str,diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rsindex 8e0849676..4b3473809 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs@@ -1,6 +1,6 @@use std::mem::size_of;-use crate::backends::common::{BufferRef, gpu_types::gemm::GemmAPrologueKind};+use crate::backends::common::BufferRef;#[derive(Clone, Copy, PartialEq)]pub enum Int8CodeLayout {@@ -58,17 +58,9 @@ pub enum MatmulA<T: BufferRef> {scale_group_size: u32,code_layout: Int8CodeLayout,},-}--impl<T: BufferRef> MatmulA<T> {- pub fn prologue_kind(&self) -> GemmAPrologueKind {- match self {- Self::FullPrecision {- ..- } => GemmAPrologueKind::FullPrecision,- Self::Int8Symmetric {- ..- } => GemmAPrologueKind::Int8Symmetric,- }- }+ Trellis {+ values: T,+ column_group_sums: T,+ scales: T,+ },}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rsindex 24bb89d24..8cb287064 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs@@ -7,11 +7,24 @@ use crate::{data_type::DataType,};+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct TrellisFormat {+ pub vector_width: u32,+ pub transition_bits: u32,+ pub restart_columns: Option<u32>,+}+pub enum MatmulB<TB: BufferRef> {FullPrecision {b: TB,},Quantized(QuantizedB<TB>),+ Trellis {+ codes: TB,+ row_scales: TB,+ codebook: TB,+ format: TrellisFormat,+ },}pub struct QuantizedB<TB: BufferRef> {@@ -82,10 +95,17 @@ impl<TB: BufferRef> QuantizedB<TB> {}impl<TB: BufferRef> MatmulB<TB> {+ pub fn is_trellis(&self) -> bool {+ matches!(self, Self::Trellis { .. })+ }+pub fn quantized(&self) -> Option<&QuantizedB<TB>> {match self {Self::FullPrecision {..+ }+ | Self::Trellis {+ ..} => None,Self::Quantized(quantized) => Some(quantized),}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rsindex aab560dee..af2eef3ba 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs@@ -4,6 +4,7 @@ mod error;mod kernel;mod matmul_a;mod matmul_b;+mod output;pub mod routing;pub use arguments::MatmulArguments;@@ -11,7 +12,8 @@ pub use d_ops::MatmulDOps;pub use error::MatmulError;pub use kernel::MatmulKernel;pub use matmul_a::{Int8CodeLayout, MatmulA};-pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};+pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection, TrellisFormat};+pub use output::MatmulOutput;pub use routing::{ActivationFormat, MatmulShape};use crate::backends::common::gpu_types::{QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, QuantizationMode};diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rsnew file mode 100644index 000000000..c7cb76391--- /dev/null+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs@@ -0,0 +1,38 @@+use super::{MatmulDOps, MatmulError};+use crate::backends::common::{Backend, BufferMut};++pub struct MatmulOutput<'d, B: Backend, T: BufferMut<Backend = B>> {+ pub values: T,+ /// Elements between output rows; `None` means the logical output width `n`.+ pub row_stride: Option<u32>,+ pub ops: MatmulDOps<'d, B>,+}++impl<'d, B: Backend, T: BufferMut<Backend = B>> MatmulOutput<'d, B, T> {+ pub fn new(+ values: T,+ ops: MatmulDOps<'d, B>,+ ) -> Self {+ Self {+ values,+ row_stride: None,+ ops,+ }+ }++ pub(crate) fn into_contiguous(+ self,+ columns: u32,+ path: &'static str,+ ) -> Result<(T, MatmulDOps<'d, B>), MatmulError<B>> {+ let stride = self.row_stride.unwrap_or(columns);+ if stride != columns {+ return Err(MatmulError::UnsupportedOutputStride {+ path,+ stride,+ columns,+ });+ }+ Ok((self.values, self.ops))+ }+}diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rsindex 658efa276..98d2df91d 100644--- a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs@@ -18,6 +18,7 @@ pub struct MatmulShape {pub b_transpose: bool,pub b_leading_dimension: Option<u32>,pub b_prologue: GemmBPrologueKind,+ pub b_is_trellis: bool,pub b_bits: Option<u32>,pub b_group_size: Option<u32>,pub signed_codes: bool,@@ -45,13 +46,14 @@ impl MatmulShape {b_transpose: arguments.b_transpose,b_leading_dimension: arguments.b_leading_dimension,b_prologue: arguments.b.b_prologue(),+ b_is_trellis: arguments.b.is_trellis(),b_bits: arguments.b.bits_per_b(),b_group_size: arguments.b.group_size(),signed_codes: arguments.b.signed_codes(),a_full_precision: matches!(arguments.a, MatmulA::FullPrecision { .. }),gathered: arguments.gather_indices.is_some(),params_layout: arguments.b.quant_params_layout(),- d_transform: arguments.d_transform.mask(),+ d_transform: arguments.output.ops.mask(),}}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex 5273d789b..0e72889e0 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -67,24 +67,24 @@ impl MatmulKernel for MatmulCpuKernel {>,command_buffer: &mut CpuCommandBufferEncoding,) -> Result<(), CpuError> {- let output_scale = arguments.d_transform.ab_scale;- let accumulate = arguments.d_transform.accumulate;- let bias_buffer = arguments.d_transform.bias;- let post_rht = arguments.d_transform.rht_factors;- let soft_cap = arguments.d_transform.soft_cap;-let MatmulArguments {a,b,b_leading_dimension,b_transpose,- mut d,+ output,m,n,k,gather_indices,..} = arguments;+ let (mut d, ops) = output.into_contiguous(n, "CpuMatmul")?;+ let output_scale = ops.ab_scale;+ let accumulate = ops.accumulate;+ let bias_buffer = ops.bias;+ let post_rht = ops.rht_factors;+ let soft_cap = ops.soft_cap;let m_u = m as usize;let n_u = n as usize;@@ -146,6 +146,14 @@ impl MatmulKernel for MatmulCpuKernel {code_layout,}},+ MatmulA::Trellis {+ ..+ } => {+ return Err(MatmulError::UnsupportedLayout {+ path: "CpuMatmul",+ }+ .into());+ },};let bias_ptr = bias_buffer.map(|bias| SendPtr(bias.cpu_ptr().as_ptr().cast::<u8>().cast_const()));let gather_ptr = gather_indices.map(|indices| {diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rsindex e6674ab1f..9ca424454 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs@@ -57,6 +57,11 @@ impl WeightData {transpose: b_transpose,})},+ MatmulB::Trellis {+ ..+ } => Err(MatmulError::UnsupportedLayout {+ path: "CpuMatmul",+ }),MatmulB::Quantized(quantized) => Ok(WeightData::Quantized {weights: buffer_ptr(quantized.codes),scales: buffer_ptr(quantized.scales),diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rsindex dec62f708..bf19f9ea0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs@@ -6,7 +6,7 @@ use crate::{Backend, BufferRef, CommandBufferEncoding, Kernels,kernel::{AttentionArguments, AttentionKernelConfig, SoftmaxKernel,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},},},metal::{@@ -119,11 +119,13 @@ impl AttentionFallback {},b_leading_dimension: Some(self.num_groups * self.head_dim),b_transpose: true,- d: &mut group_scores,- d_transform: MatmulDOps {- ab_scale: scale,- ..MatmulDOps::none()- },+ output: MatmulOutput::new(+ &mut group_scores,+ MatmulDOps {+ ab_scale: scale,+ ..MatmulDOps::none()+ },+ ),gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,m: gqa_factor * suffix_length,n: sequence_length,@@ -168,8 +170,7 @@ impl AttentionFallback {},b_leading_dimension: Some(self.num_groups * self.head_dim),b_transpose: false,- d: &mut group_output,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(&mut group_output, MatmulDOps::none()),gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,m: gqa_factor * suffix_length,n: self.head_dim,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 301c78864..beee31ab8 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -109,15 +109,15 @@ struct MxuMmaCore {? SIMDGROUP_BLOCK_N: short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));- schedules::TileContext tile_context;- tile_context.simdgroup_limit_m = simdgroup_limit_m;- tile_context.simdgroup_limit_n = simdgroup_limit_n;- tile_context.block_col = block_col;- tile_context.tile_col_offset = tile_col_offset;- tile_context.tile_block_cols =- ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start)));- tile_context.k_offset = k_offset;- tile_context.abs_row_base = uint(geometry.block_row_start) + tile_row_offset;+ const schedules::TileContext tile_context{+ .simdgroup_limit_m = simdgroup_limit_m,+ .simdgroup_limit_n = simdgroup_limit_n,+ .block_col = block_col,+ .tile_col_offset = tile_col_offset,+ .tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start))),+ .k_offset = k_offset,+ .abs_row_base = uint(geometry.block_row_start) + tile_row_offset,+ };const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.hnew file mode 100644index 000000000..443f847e6--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h@@ -0,0 +1,140 @@+#pragma once++#include <metal_stdlib>++#include "../../../../common/defines.h"+#include "../../../../generated/gemm.h"+#include "cursor.h"++using namespace metal;++namespace uzu {+namespace gemm {++namespace trellis {++struct GemmTrellisFormat {+ uint32_t vector_width;+ uint32_t transition_bits;+ uint32_t restart_columns;+};++UZU_CONST uint K_STEP = 64, WEIGHT_BIAS = 54;+UZU_CONST uint STATE_BITS = 16, BYTE_BITS = 8, V4_VECTOR_WIDTH = 4, V2T6_TRANSITION_BITS = 6;++UZU_CONST uint HASH_MULTIPLIER = 0xCFCCB83Fu, HASH_INCREMENT = 0x584B4AA3u;+UZU_CONST uint HASH_AVALANCHE_MULTIPLIER = 0x85EBCA6Bu, HASH_FOLD_BITS = 16;++UZU_CONST uint TWO_BIT_FIELD_SHIFT = 2, NIBBLE_SHIFT = 4, FIELD_SUM_SHIFT = 3;+UZU_CONST uint TWO_BIT_FIELDS_MASK = 0x33333333u, BYTE_NIBBLES_MASK = 0x0F0F0F0Fu;+UZU_CONST uint FIELD_PAIR_SUM_MULTIPLIER = 3, DITHER_MULTIPLIER = 3;++// Row layout+static METAL_FUNC uint block_bytes(const GemmTrellisFormat format, const uint block_columns) {+ const uint state_count = block_columns / format.vector_width;+ const uint transition_count = state_count - 1;+ return (STATE_BITS + transition_count * format.transition_bits + BYTE_BITS - 1) / BYTE_BITS;+}++static METAL_FUNC uint row_bytes(const GemmTrellisFormat format, const uint code_columns) {+ const uint block_columns = format.restart_columns == 0 ? code_columns : format.restart_columns;+ return code_columns / block_columns * block_bytes(format, block_columns);+}++// MSB-first codes. code_column must be a multiple of 4.+static METAL_FUNC uint2+states_at(const GemmTrellisFormat format, const device uchar* code_row, const uint code_column) {+ if (format.vector_width == V4_VECTOR_WIDTH) {+ const device uchar* block_codes = code_row;+ uint block_column = code_column;+ if (format.restart_columns != 0) {+ block_codes += code_column / format.restart_columns * block_bytes(format, format.restart_columns);+ block_column %= format.restart_columns;+ }+ const device uchar* state_bytes = block_codes + block_column / V4_VECTOR_WIDTH;+ return uint2(uint(state_bytes[0]) << BYTE_BITS | state_bytes[1], 0);+ }+ const uint first_bit = code_column / format.vector_width * format.transition_bits;+ const device uchar* bytes = code_row + first_bit / BYTE_BITS;+ const uint shift = first_bit % BYTE_BITS;+ const uint fourth_byte = format.transition_bits == V2T6_TRANSITION_BITS ? uint(bytes[3]) : 0;+ const uint window = uint(bytes[0]) << 24 | uint(bytes[1]) << 16 | uint(bytes[2]) << 8 | fourth_byte;+ const uint state_mask = (1u << STATE_BITS) - 1;+ const uint first_state = (window >> (16 - shift)) & state_mask;+ const uint second_state = (window >> (16 - format.transition_bits - shift)) & state_mask;+ return uint2(first_state, second_state);+}++// State -> hash -> biased weights+static METAL_FUNC uint hash_without_final_fold(const uint code_state) {+ uint hash = code_state * HASH_MULTIPLIER + HASH_INCREMENT;+ hash ^= hash >> HASH_FOLD_BITS;+ return hash * HASH_AVALANCHE_MULTIPLIER;+}++static METAL_FUNC uint final_fold(const uint hash) { return hash ^ (hash >> HASH_FOLD_BITS); }++// Per byte b: weight + WEIGHT_BIAS = 8 * (sum of b's four 2-bit fields) + ((3 * (b & 15)) & 15).+static METAL_FUNC uint biased_weights(const uint hash_bytes) {+ const uint nibble_field_sums =+ hash_bytes - FIELD_PAIR_SUM_MULTIPLIER * ((hash_bytes >> TWO_BIT_FIELD_SHIFT) & TWO_BIT_FIELDS_MASK);+ const uint byte_field_sums = (nibble_field_sums + (nibble_field_sums >> NIBBLE_SHIFT)) & BYTE_NIBBLES_MASK;+ const uint low_nibble_dither = ((hash_bytes & BYTE_NIBBLES_MASK) * DITHER_MULTIPLIER) & BYTE_NIBBLES_MASK;+ return (byte_field_sums << FIELD_SUM_SHIFT) + low_nibble_dither;+}++static METAL_FUNC uint+decode_biased_weights(const GemmTrellisFormat format, const device uchar* code_row, const uint code_column) {+ const uint2 code_states = states_at(format, code_row, code_column);+ const uint first_hash = hash_without_final_fold(code_states.x);+ if (format.vector_width == V4_VECTOR_WIDTH) {+ return biased_weights(final_fold(first_hash));+ }+ const uint second_hash = hash_without_final_fold(code_states.y);+ const uint low_halves = insert_bits(first_hash, second_hash, HASH_FOLD_BITS, HASH_FOLD_BITS);+ const uint high_halves = insert_bits(second_hash, first_hash >> HASH_FOLD_BITS, 0, HASH_FOLD_BITS);+ return biased_weights(low_halves ^ high_halves);+}++} // namespace trellis++namespace quantized {++template <typename Fragment, bool ALIGNED>+struct TrellisCursor {+ using Ops = typename Fragment::FragmentOpsType;+ UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);++ const device uint8_t* code_rows;+ uint code_row_stride_bytes;+ short tile_row_limit;+ short2 position;+ uint k_offset;+ trellis::GemmTrellisFormat format;++ METAL_FUNC Fragment load(const uint) const thread {+ const short row_limit = tile_row_limit - position.y;+ Fragment tile;+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {+ const bool live = ALIGNED || row_offset < row_limit;+ const device uint8_t* code_row = code_rows + int(row_offset) * int(code_row_stride_bytes);+ METAL_PRAGMA_UNROLL+ for (ushort k_fragment = 0; k_fragment < Fragment::COL_FRAGMENTS; ++k_fragment) {+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, k_fragment))[row_slot] =+ live ? trellis::decode_biased_weights(+ format,+ code_row,+ k_offset + uint(k_fragment * Ops::FRAGMENT_ROWS) + uint(position.x)+ )+ : 0u;+ }+ });+ return tile;+ }++ METAL_FUNC void advance() thread { k_offset += uint(BLOCK_K); }+};++} // namespace quantized+} // namespace gemm+} // namespace uzudiff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metalnew file mode 100644index 000000000..137171472--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal@@ -0,0 +1,194 @@+#include "../../common/dsl.h"+#include "../../common/defines.h"+#include "../../common/thread_context.h"+#include "../generated/gemm.h"++#include "common/gemm_tiling.h"+#include "common/mxu_mma_core.h"+#include "common/quantized/trellis.h"++using namespace metal;+using namespace uzu::gemm;++#define REDUCE_THREADS 256+#define REDUCE_VECTOR_WIDTH 4++namespace {++struct TrellisEpilogue {+ const device float* scale_and_offsets;+ const device float4* group_sums;+ const device float* activation_scales;+ const device float* row_scales;+ device bfloat* output;+ uint output_row_stride;++ METAL_FUNC void store(const int biased_dot, const uint token, const uint row) const {+ const float4 group_sum = group_sums[token];+ const int level_dot =+ biased_dot - int(trellis::WEIGHT_BIAS) * int(group_sum.x + group_sum.y + group_sum.z + group_sum.w);+ const float activation_scale = activation_scales[token];+ const float4 offsets =+ float4(scale_and_offsets[1], scale_and_offsets[2], scale_and_offsets[3], scale_and_offsets[4]);+ const float dot = float(level_dot) * scale_and_offsets[0] + metal::dot(group_sum, offsets);+ output[token * output_row_stride + row] = bfloat(dot * row_scales[row] * activation_scale);+ }+};++} // namespace++template <GemmTiling GEMM_TILING>+VARIANTS(+ GEMM_TILING,+ GemmTiling::Tile16x32x256_Simdgroups1x1,+ GemmTiling::Tile64x64x256_Simdgroups2x2,+ GemmTiling::Tile128x128x256_Simdgroups4x4)+KERNEL(GemmTrellis)(+ const device int8_t* activations,+ const device float4* column_group_sums,+ const device float* activation_scales,+ const device uint8_t* codes,+ const device float* row_scales,+ const device float* scale_and_offsets,+ device bfloat* output,+ const constant uzu::matmul::GemmParams* params,+ const constant uint& group_count_x,+ const constant uint& group_count_y,+ const constant uint& group_count_z,+ const GemmAlignment alignment SPECIALIZE,+ const uint trellis_vector_width SPECIALIZE,+ const uint trellis_transition_bits SPECIALIZE,+ const uint trellis_restart_columns SPECIALIZE,+ const uint group_x GROUPS(group_count_x),+ const uint group_y GROUPS(group_count_y),+ const uint group_z GROUPS(group_count_z),+ const uint thread_x THREADS(METAL_SIMD_SIZE),+ const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),+ const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),+ const ThreadContext thread_context+) {+ (void)thread_x;+ (void)thread_y;+ (void)thread_z;++ using LeftOperand =+ operands::LeftOperandFor<GemmAPrologueKind::Int8Symmetric, bfloat, ushort(trellis::K_STEP), false>;+ // A stock int8 right operand, only for the tile constants of `Core`.+ using RightOperand =+ operands::RightOperandFor<GemmBPrologueKind::ScaleSymmetricDequant, ushort(8), ushort(trellis::K_STEP), bfloat>;+ using Core = MxuMmaCore<bfloat, GEMM_TILING, true, LeftOperand, RightOperand>;+ using Ops = typename Core::FragmentOps;+ using Products = uzu::matmul::Fragment<int, Core::TILES_M, Core::TILES_N, Ops>;+ using RightFragment = uzu::matmul::Fragment<int8_t, Core::TILES_N, Core::TILES_K, Ops, uzu::matmul::ReadDirect, true>;+ const trellis::GemmTrellisFormat trellis_format{+ trellis_vector_width,+ trellis_transition_bits,+ trellis_restart_columns+ };++ const uint simdgroup = thread_context.simdgroup_index;+ const uint token_base =+ group_y * Core::THREADGROUP_BLOCK_M + Core::SIMDGROUP_BLOCK_M * (simdgroup / Core::SIMDGROUPS_PER_COLUMN);+ const uint row_base =+ group_x * Core::THREADGROUP_BLOCK_N + Core::SIMDGROUP_BLOCK_N * (simdgroup % Core::SIMDGROUPS_PER_COLUMN);+ const short token_end = short(min(int(Core::SIMDGROUP_BLOCK_M), int(params->M) - int(token_base)));+ const short row_end = short(min(int(Core::SIMDGROUP_BLOCK_N), int(params->N) - int(row_base)));+ const schedules::TileContext tile_context{+ .simdgroup_limit_m = token_end,+ .simdgroup_limit_n = row_end,+ .k_offset = group_z * params->aligned_inner_iterations * trellis::K_STEP,+ .abs_row_base = token_base,+ };++ const uint code_row_bytes = trellis::row_bytes(trellis_format, params->K);+ const short2 position = Ops::get_position(thread_context.simd_lane_id);+ const auto left_storage = operands::pack_left<LeftOperand, bfloat>(nullptr, activations, nullptr, nullptr);++ Products products;+ products.clear();+ uzu::dispatch_bool(alignment.contains(GemmAlignment::M) || token_end == Core::SIMDGROUP_BLOCK_M, [&](auto aligned_m) {+ uzu::dispatch_bool(alignment.contains(GemmAlignment::N) || row_end == Core::SIMDGROUP_BLOCK_N, [&](auto aligned_n) {+ auto left = quantized::make_left_cursor<true, Core, LeftOperand, aligned_m.value>(+ left_storage,+ params,+ tile_context,+ thread_context+ );+ quantized::TrellisCursor<RightFragment, aligned_n.value> trellis_cursor{+ codes + size_t(row_base + position.y) * code_row_bytes,+ code_row_bytes,+ row_end,+ position,+ tile_context.k_offset,+ trellis_format+ };+ const int k_chunks = token_end > 0 && row_end > 0+ ? int(params->aligned_inner_iterations) * int(trellis::K_STEP / Core::SIMDGROUP_BLOCK_K)+ : 0;+ METAL_PRAGMA_NO_UNROLL+ for (int chunk = 0; chunk < k_chunks; ++chunk) {+ auto left_tile = left.load(0u);+ auto right_tile = trellis_cursor.load(0u);+ uzu::matmul::fragment_mma(products, left_tile, right_tile);+ left.advance();+ trellis_cursor.advance();+ }+ });+ });++ const bool write_split_k_partials = params->aligned_inner_iterations * trellis::K_STEP < params->K;+ const size_t partial_sum_offset = size_t(group_z) * params->M * params->N;+ const TrellisEpilogue epilogue{+ scale_and_offsets,+ column_group_sums,+ activation_scales,+ row_scales,+ output,+ params->leading_dimension_d+ };+ products.map_coords(thread_context.simd_lane_id, [&](short token_offset, short row_offset, int value) {+ if (token_offset < token_end && row_offset < row_end) {+ const uint token = token_base + uint(token_offset);+ const uint row = row_base + uint(row_offset);+ if (write_split_k_partials) {+ device int* partial_sums = reinterpret_cast<device int*>(output) + partial_sum_offset;+ partial_sums[size_t(token) * params->N + row] = value;+ } else {+ epilogue.store(value, token, row);+ }+ }+ return value;+ });+}++KERNEL(GemmTrellisReduce)(+ const device int4* partial_sums,+ const device float4* column_group_sums,+ const device float* activation_scales,+ const device float* row_scales,+ const device float* scale_and_offsets,+ device bfloat* output,+ const constant uint& token_count,+ const constant uint& row_count,+ const constant uint& partition_count,+ const constant uint& output_stride,+ const uint threadgroup_index GROUPS((token_count * (row_count / REDUCE_VECTOR_WIDTH)).div_ceil(REDUCE_THREADS)),+ const uint thread_index_in_threadgroup THREADS(REDUCE_THREADS)+) {+ const uint row_groups = row_count / REDUCE_VECTOR_WIDTH;+ const uint token_row_group = threadgroup_index * REDUCE_THREADS + thread_index_in_threadgroup;+ if (token_row_group >= token_count * row_groups) {+ return;+ }+ int4 partial_sum = int4(0);+ for (uint partition = 0u; partition < partition_count; ++partition) {+ partial_sum += partial_sums[partition * token_count * row_groups + token_row_group];+ }+ const uint token = token_row_group / row_groups;+ const uint first_row = token_row_group % row_groups * REDUCE_VECTOR_WIDTH;+ const TrellisEpilogue+ epilogue{scale_and_offsets, column_group_sums, activation_scales, row_scales, output, output_stride};+ for (uint index = 0u; index < REDUCE_VECTOR_WIDTH; ++index) {+ epilogue.store(partial_sum[index], token, first_row + index);+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 0854876ad..fe4ae2bdf 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -5,6 +5,7 @@ use super::{GemmEngine, GemmPlan,selection::{GemmProblem, outer_block_k},specialization::GemmSpecialization,+ trellis::TrellisGemm,};use crate::{backends::{@@ -15,7 +16,7 @@ use crate::{gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform},},kernel::matmul::{- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulError, MatmulShape,+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulOutput, MatmulShape,QuantParamsLayout, QuantParamsStrides, QuantizedB,},},@@ -34,6 +35,7 @@ pub struct GemmKernel {weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,+ trellis: TrellisGemm,kernels: HashMap<GemmSpecialization, GemmMetalKernel>,split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,}@@ -48,6 +50,7 @@ impl GemmKernel {weights_data_type,input_data_type,output_data_type,+ trellis: TrellisGemm::default(),kernels: HashMap::new(),split_k_reduce: HashMap::new(),}@@ -138,12 +141,22 @@ impl GemmKernel {command_buffer: &mut MetalCommandBufferEncoding,) -> Result<(), MetalError> {let shape = MatmulShape::from_arguments(&arguments);+ // TODO: remove after GatherGEMM is supported+ if arguments.gather_indices.is_some() {+ return Err(MetalError::KernelDispatchFailed(+ format!(+ "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",+ arguments.m, arguments.n+ )+ .into(),+ ));+ }self.problem(shape, command_buffer.context()).validate_engine(plan.engine).map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;if !matches!(arguments.b, MatmulB::FullPrecision { .. })- && arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE)+ && arguments.output.ops.mask().contains(GemmDTransform::ACCUMULATE){return Err(MatmulError::UnsupportedDOp {bit: GemmDTransform::ACCUMULATE,@@ -152,28 +165,37 @@ impl GemmKernel {.into());}- let MatmulArguments {- a,- b,- d,- d_transform,- ..- } = arguments;- match b {+ match arguments.b {MatmulB::FullPrecision {b: weights,- } => self.encode_weights(a, weights, None, d, d_transform, shape, plan, output_work, command_buffer),+ } => self.encode_weights(+ arguments.a,+ weights,+ None,+ arguments.output,+ shape,+ plan,+ output_work,+ command_buffer,+ ),MatmulB::Quantized(quantized) => self.encode_weights(- a,+ arguments.a,quantized.codes,Some(quantized),- d,- d_transform,+ arguments.output,shape,plan,output_work,command_buffer,),+ MatmulB::Trellis {+ ..+ } => {+ if self.output_data_type != DataType::BF16 {+ return Err(MatmulError::UnsupportedDataType(self.output_data_type).into());+ }+ self.trellis.encode_plan(arguments, plan, command_buffer)+ },}}@@ -182,14 +204,14 @@ impl GemmKernel {a: MatmulA<impl BufferRef<Backend = Metal>>,weights: WB,quantized: Option<QuantizedB<WB>>,- mut d: impl BufferMut<Backend = Metal>,- d_transform: MatmulDOps<'_, Metal>,+ output: MatmulOutput<'_, Metal, impl BufferMut<Backend = Metal>>,shape: MatmulShape,plan: GemmPlan,output_work: &MatmulOutputWork,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<(), MetalError> {let (m, n, k) = (shape.m, shape.n, shape.k);+ let (mut d, d_transform) = output.into_contiguous(n, "Gemm")?;let ab_scale = d_transform.ab_scale;let output_bias = d_transform.bias;let rht_factors = d_transform.rht_factors;@@ -221,12 +243,11 @@ impl GemmKernel {quantized.zero_point_strides(),),};- let a_prologue = a.prologue_kind();- let (a_full_precision, a_int8, a_scales, a_group_sums, a_group_size) = match &a {+ let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match &a {MatmulA::FullPrecision {values,offset,- } => (Some(values.subrange(*offset..)), None, None, None, None),+ } => (Some(values.subrange(*offset..)), None, None, None, GemmAPrologueKind::FullPrecision, None),MatmulA::Int8Symmetric {values,scales: activation_scales,@@ -248,7 +269,22 @@ impl GemmKernel {}.into());}- (None, Some(*values), Some(*activation_scales), *activation_group_sums, Some(*scale_group_size))+ (+ None,+ Some(*values),+ Some(*activation_scales),+ *activation_group_sums,+ GemmAPrologueKind::Int8Symmetric,+ Some(*scale_group_size),+ )+ },+ MatmulA::Trellis {+ ..+ } => {+ return Err(MatmulError::UnsupportedLayout {+ path: "Gemm",+ }+ .into());},};@@ -368,6 +404,14 @@ impl GemmKernel {} => {(None, Some(values), Some(scales), group_sums, GemmAPrologueKind::Int8Symmetric, Some(scale_group_size))},+ MatmulA::Trellis {+ ..+ } => {+ return Err(MatmulError::UnsupportedLayout {+ path: "Gemm",+ }+ .into());+ },};let tiling = plan.tiling;let split_k = plan.split_k;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rsindex 2350fd612..d45b0d0c0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -3,6 +3,7 @@ mod kernel;mod policy;mod selection;mod specialization;+mod trellis;pub use error::GemmSpecializationError;pub use kernel::GemmKernel;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex 7a439b16e..9e5ba1d04 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -27,9 +27,9 @@ const SIMDGROUP_QUANT_LARGE_M_MIN: u32 = 64;const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;const SPLIT_K_TARGET_TILES_FP: u32 = 512;-const SPLIT_K_TARGET_TILES_A8: u32 = 256;-const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;-const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;+pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;+pub(super) const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;+pub(super) const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;const SPLIT_K_TARGET_TILES_A8_TILE32_W8: u32 = 1024;fn bucket(diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rsindex 6e5c0c8ae..b5c49acd9 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs@@ -10,6 +10,8 @@ use crate::{data_type::DataType,};+pub(super) const TRELLIS_K_STEP: u32 = 64;+#[derive(Clone, Copy)]pub struct GemmProblem {shape: MatmulShape,@@ -23,8 +25,8 @@ pub struct GemmProblem {pub(super) enum GemmPlanError {#[error("MXU engine is not available for this GEMM")]MxuUnavailable,- #[error("quantized GEMM requires transposed contiguous B")]- UnsupportedQuantLayout,+ #[error("GEMM input layout is unsupported: {0}")]+ UnsupportedLayout(&'static str),}impl GemmProblem {@@ -45,6 +47,9 @@ impl GemmProblem {}pub fn select_plan(self) -> GemmPlan {+ if self.shape.b_is_trellis {+ return select_trellis_plan(self.shape.m, self.shape.n, self.shape.k);+ }let engine = if self.supports_mxu && mxu_is_eligible(self.shape) {GemmEngine::Mxu} else {@@ -58,6 +63,11 @@ impl GemmProblem {self,engine: GemmEngine,) -> Result<GemmPlan, GemmPlanError> {+ if self.shape.b_is_trellis {+ let mut plan = select_trellis_plan(self.shape.m, self.shape.n, self.shape.k);+ plan.engine = engine;+ return Ok(plan);+ }self.validate_engine(engine)?;Ok(self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family)))}@@ -70,7 +80,10 @@ impl GemmProblem {return Err(GemmPlanError::MxuUnavailable);}if self.shape.is_quant() && (!self.shape.b_transpose || self.shape.b_leading_dimension.is_some()) {- return Err(GemmPlanError::UnsupportedQuantLayout);+ return Err(GemmPlanError::UnsupportedLayout("quantized weights require transposed contiguous B"));+ }+ if self.shape.b_is_trellis && !self.shape.k.is_multiple_of(TRELLIS_K_STEP) {+ return Err(GemmPlanError::UnsupportedLayout("Trellis K must be divisible by 64"));}Ok(())}@@ -140,6 +153,36 @@ impl GemmProblem {}}+pub(super) fn select_trellis_plan(+ m: u32,+ n: u32,+ k: u32,+) -> GemmPlan {+ let (tiling, target_workgroups) = match policy::mxu_mn_tile(true, m, n) {+ GemmTiling::Tile16x32x256_Simdgroups1x1 => {+ (GemmTiling::Tile16x32x256_Simdgroups1x1, policy::SPLIT_K_TARGET_TILES_A8_TILE16X32)+ },+ GemmTiling::Tile32x64x256_Simdgroups2x2 => {+ (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8_TILE32_W4)+ },+ GemmTiling::Tile128x128x256_Simdgroups4x4 => {+ (GemmTiling::Tile128x128x256_Simdgroups4x4, policy::SPLIT_K_TARGET_TILES_A8)+ },+ _ => (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8),+ };+ let output_workgroups = (n.div_ceil(tiling.block_n()) * m.div_ceil(tiling.block_m())).max(1);+ let k_steps = (k / TRELLIS_K_STEP).max(1);+ let mut split_k = (target_workgroups / output_workgroups).clamp(1, k_steps);+ while !k_steps.is_multiple_of(split_k) {+ split_k -= 1;+ }+ GemmPlan {+ engine: GemmEngine::Mxu,+ tiling,+ split_k,+ }+}+pub(super) fn outer_block_k(shape: MatmulShape,engine: GemmEngine,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rsnew file mode 100644index 000000000..0929b24b3--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs@@ -0,0 +1,145 @@+use std::collections::{HashMap, hash_map::Entry};++use super::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP};+use crate::backends::{+ common::{+ BufferMut, BufferRef, CommandBufferEncoding,+ gpu_types::{+ GemmParams,+ gemm::{GemmAlignment, GemmTiling},+ },+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, TrellisFormat},+ },+ metal::{+ Metal,+ command_buffer::MetalCommandBufferEncoding,+ error::MetalError,+ kernel::{GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel},+ },+};++#[derive(Default)]+pub(super) struct TrellisGemm {+ kernels: HashMap<(GemmTiling, TrellisFormat, GemmAlignment), GemmTrellisMetalKernel>,+ reduce_kernel: Option<GemmTrellisReduceMetalKernel>,+}++impl TrellisGemm {+ pub(super) fn encode_plan(+ &mut self,+ arguments: MatmulArguments<+ '_,+ Metal,+ impl BufferRef<Backend = Metal>,+ impl BufferRef<Backend = Metal>,+ impl BufferMut<Backend = Metal>,+ impl BufferRef<Backend = Metal>,+ >,+ plan: GemmPlan,+ command_buffer: &mut MetalCommandBufferEncoding,+ ) -> Result<(), MetalError> {+ let unsupported = || {+ MetalError::from(MatmulError::UnsupportedLayout {+ path: "GemmTrellis",+ })+ };+ let (+ MatmulA::Trellis {+ values: activations,+ column_group_sums,+ scales: activation_scales,+ },+ MatmulB::Trellis {+ codes,+ row_scales,+ codebook,+ format,+ },+ ) = (arguments.a, arguments.b)+ else {+ return Err(unsupported());+ };+ let (m, n, k) = (arguments.m, arguments.n, arguments.k);+ let output_stride = arguments.output.row_stride.unwrap_or(n);+ let (tiling, split_k) = (plan.tiling, plan.split_k);+ if plan.engine != GemmEngine::Mxu+ || !arguments.output.ops.mask().is_empty()+ || !arguments.b_transpose+ || arguments.b_leading_dimension.is_some()+ || output_stride < n+ || (split_k > 1 && !n.is_multiple_of(4))+ {+ return Err(unsupported());+ }++ let params = GemmParams {+ M: m,+ N: n,+ K: k,+ leading_dimension_a: k,+ leading_dimension_d: output_stride,+ threadgroups_per_row: n.div_ceil(tiling.block_n()),+ threadgroups_per_column: m.div_ceil(tiling.block_m()),+ aligned_inner_iterations: k / split_k / K_STEP,+ ..Default::default()+ };+ let alignment =+ GemmAlignment::new(m.is_multiple_of(tiling.block_m()), n.is_multiple_of(tiling.block_n()), true);+ let key = (tiling, format, alignment);+ let kernel = match self.kernels.entry(key) {+ Entry::Occupied(entry) => entry.into_mut(),+ Entry::Vacant(entry) => entry.insert(GemmTrellisMetalKernel::new(+ command_buffer.context(),+ tiling,+ alignment,+ format.vector_width,+ format.transition_bits,+ format.restart_columns.unwrap_or(0),+ )?),+ };+ macro_rules! encode_gemm {+ ($destination:expr) => {+ kernel.encode(+ activations,+ column_group_sums,+ activation_scales,+ codes,+ row_scales,+ codebook,+ $destination,+ std::slice::from_ref(¶ms),+ params.threadgroups_per_row,+ params.threadgroups_per_column,+ split_k,+ command_buffer,+ )+ };+ }+ if split_k == 1 {+ encode_gemm!(arguments.output.values);+ return Ok(());+ }++ let partial_count = split_k as usize * m as usize * n as usize;+ let mut partial_sums = command_buffer.allocate_scratch(partial_count * std::mem::size_of::<i32>())?;+ encode_gemm!(&mut partial_sums);+ let reduce_kernel = match &mut self.reduce_kernel {+ Some(kernel) => kernel,+ empty => empty.insert(GemmTrellisReduceMetalKernel::new(command_buffer.context())?),+ };+ reduce_kernel.encode(+ &partial_sums,+ column_group_sums,+ activation_scales,+ row_scales,+ codebook,+ arguments.output.values,+ m,+ n,+ split_k,+ output_stride,+ command_buffer,+ );+ Ok(())+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 1e44c109a..ff949980c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -271,27 +271,27 @@ impl GemvKernel {output_work: &MatmulOutputWork,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<(), MatmulError<Metal>> {- let ab_scale = arguments.d_transform.ab_scale;- let output_bias = arguments.d_transform.bias;- let rht_factors = arguments.d_transform.rht_factors;- let soft_cap = arguments.d_transform.soft_cap;- let deferred_factors = rht_factors.filter(|_| !specialization.fuses_rht());- let (gemv_bias, gemv_rht_factors) = if deferred_factors.is_some() {- (None, None)- } else {- (output_bias, rht_factors)- };-let MatmulArguments {a,b,- mut d,+ output,m,n,k,gather_indices,..} = arguments;+ let (mut d, ops) = output.into_contiguous(n, "Gemv")?;+ let ab_scale = ops.ab_scale;+ let output_bias = ops.bias;+ let rht_factors = ops.rht_factors;+ let soft_cap = ops.soft_cap;+ let deferred_factors = rht_factors.filter(|_| !specialization.fuses_rht());+ let (gemv_bias, gemv_rht_factors) = if deferred_factors.is_some() {+ (None, None)+ } else {+ (output_bias, rht_factors)+ };let MatmulA::FullPrecision {values: a,offset: a_offset,@@ -364,6 +364,13 @@ impl GemvKernel {soft_cap,command_buffer,),+ MatmulB::Trellis {+ ..+ } => {+ return Err(MatmulError::UnsupportedLayout {+ path: "Gemv",+ });+ },}if let Some(factors) = deferred_factors {diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rsindex b6e8fddda..4257c9dda 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs@@ -31,7 +31,7 @@ use crate::{pub struct MatmulMetalKernel {gemv: GemvKernel,- pub gemm: GemmKernel,+ gemm: GemmKernel,output_work: MatmulOutputWork,weights_data_type: DataType,input_data_type: DataType,@@ -130,6 +130,10 @@ impl MatmulMetalKernel {let all_bf16 = weights_data_type == DataType::BF16&& input_data_type == DataType::BF16&& output_data_type == DataType::BF16;+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);+ if shape.b_is_trellis {+ return MatmulDispatch::Gemm(problem.select_plan());+ }if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {return match route {QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(@@ -147,7 +151,6 @@ impl MatmulMetalKernel {gpu_core_count,apple_gpu_family,);- let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);let plan = problem.select_plan();match gemv {None => MatmulDispatch::Gemm(plan),@@ -274,24 +277,12 @@ impl MatmulKernel for MatmulMetalKernel {command_buffer: &mut MetalCommandBufferEncoding,) -> Result<(), MetalError> {let shape = MatmulShape::from_arguments(&arguments);- let plan = match self.select_dispatch(&shape, command_buffer.context()) {+ match self.select_dispatch(&shape, command_buffer.context()) {MatmulDispatch::Gemv(gemv) => {- return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from);+ self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from)},- MatmulDispatch::Gemm(plan) => plan,- };-- // TODO: remove after GatherGEMM is supported- if arguments.gather_indices.is_some() {- return Err(MetalError::KernelDispatchFailed(- format!(- "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",- arguments.m, arguments.n- )- .into(),- ));+ MatmulDispatch::Gemm(plan) => self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer),}- self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)}}diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex 3ed2ccabf..fd218c54a 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -7,7 +7,7 @@ use crate::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,kernel::{LogitTransformKernel,- matmul::{MatmulA, MatmulArguments, MatmulDOps, MatmulKernel},+ matmul::{MatmulA, MatmulArguments, MatmulDOps, MatmulKernel, MatmulOutput},},},config::{embedding::AnyEmbeddingConfig, weight_matrix::AnyWeightMatrixSpec},@@ -218,8 +218,7 @@ impl<B: Backend> Embedding<B> {b: table.as_matrix().expect("tied embedding tables are matrices").matmul_b(),b_leading_dimension: None,b_transpose: true,- d: &mut output,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(&mut output, MatmulDOps::none()),gather_indices,m: batch_dim,n: output_dim,diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rsindex 23219f816..4f7f6a9ad 100644--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs@@ -7,7 +7,10 @@ use crate::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding,kernel::{ActivationQuantization, Kernels,- matmul::{ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulShape},+ matmul::{+ ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,+ MatmulShape,+ },},},config::weight_matrix::{AnyWeightMatrixSpec, Layout},@@ -131,8 +134,7 @@ impl<B: Backend> LinearMatmul<B> {b: self.matmul_b(),b_leading_dimension: None,b_transpose: true,- d: &mut output,- d_transform: self.d_ops(),+ output: MatmulOutput::new(&mut output, self.d_ops()),gather_indices,m: batch_dim,n: output_dim,@@ -157,6 +159,7 @@ impl<B: Backend> LinearMatmul<B> {b_transpose: true,b_leading_dimension: None,b_prologue: b.b_prologue(),+ b_is_trellis: b.is_trellis(),b_bits: b.bits_per_b(),b_group_size: b.group_size(),signed_codes: b.signed_codes(),diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rsindex bb818d607..d4faa24b9 100644--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs@@ -8,7 +8,7 @@ use crate::{gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,kernel::{ActivationTransform, Kernels,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},},},config::weight_matrix::{AnyWeightMatrixSpec, hybrid_spec::IncoherenceProcessingMode, low_rank_spec::LowRankSpec},@@ -193,8 +193,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {},b_leading_dimension: None,b_transpose: true,- d: &mut intermediate,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(&mut intermediate, MatmulDOps::none()),gather_indices: None::<&B::ScratchBuffer>,m: batch_dim,n: self.lora_rank,@@ -235,11 +234,13 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {},b_leading_dimension: None,b_transpose: true,- d: &mut output,- d_transform: MatmulDOps {- accumulate: true,- ..MatmulDOps::none()- },+ output: MatmulOutput::new(+ &mut output,+ MatmulDOps {+ accumulate: true,+ ..MatmulDOps::none()+ },+ ),gather_indices: None::<&B::ScratchBuffer>,m: batch_dim,n: self.output_dim,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rsindex 962552ffc..657f20156 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs@@ -15,8 +15,8 @@ use crate::{ActivationQuantization, ActivationTransform, Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,matmul::{- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, QuantParams,- QuantParamsLayout, QuantizedB, QuantizedCorrection,+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,+ QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection,},},},@@ -148,8 +148,7 @@ impl BenchmarkData {}),b_leading_dimension: None,b_transpose: true,- d: output,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(output, MatmulDOps::none()),gather_indices: None,m: self.m,n: self.n,@@ -204,8 +203,7 @@ fn encode_step(}),b_leading_dimension: None,b_transpose: true,- d: output,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(output, MatmulDOps::none()),gather_indices: None,m: data.m,n: data.n,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rsindex 551730e75..a9448d97f 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs@@ -11,7 +11,7 @@ use crate::{Backend,kernel::{Kernels,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},},},metal::{GemmEngine, Metal},@@ -63,8 +63,7 @@ fn bench_gemm(c: &mut Criterion) {},b_leading_dimension: None,b_transpose: true,- d: &mut d,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(&mut d, MatmulDOps::none()),gather_indices: None::<&<Metal as Backend>::GlobalBuffer>,m,n,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rsindex 2ef96a40b..2924eacf8 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs@@ -13,7 +13,7 @@ use crate::{gpu_types::QuantizationMethod,kernel::{Kernels,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},},},cpu::Cpu,@@ -88,11 +88,13 @@ fn run_gemv<B: Backend, T: ArrayElement + Float>(b,b_leading_dimension: None,b_transpose: true,- d: &mut d,- d_transform: MatmulDOps {- soft_cap,- ..MatmulDOps::none()- },+ output: MatmulOutput::new(+ &mut d,+ MatmulDOps {+ soft_cap,+ ..MatmulDOps::none()+ },+ ),gather_indices,m: m as u32,n: n_out as u32,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rsindex ab1716ec5..fe5ec76ae 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs@@ -6,3 +6,4 @@ mod quant_dispatch_test;mod quant_gemm_bench;mod quant_gemv_bench;mod qwen3_bench;+mod trellis_gemm_test;diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rsindex 728ba43de..494469be9 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs@@ -21,7 +21,10 @@ use crate::{kernel::{Kernels,activation_transform::ACTIVATION_SCALE_GROUP_SIZE,- matmul::{MatmulDOps, MatmulError, MatmulKernel, QuantParams, QuantParamsLayout, QuantParamsStrides},+ matmul::{+ MatmulDOps, MatmulError, MatmulKernel, MatmulOutput, QuantParams, QuantParamsLayout,+ QuantParamsStrides,+ },},},cpu::Cpu,@@ -207,7 +210,7 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {let mut command_buffer = context.create_command_buffer(None, None).unwrap();let mut args = quant_arguments(&mut buffers, &input);- args.d_transform = MatmulDOps {+ args.output.ops = MatmulDOps {bias: Some(&bias_pp_buf),..MatmulDOps::none()};@@ -257,7 +260,7 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {let mut command_buffer = context.create_command_buffer(None, None).unwrap();let mut args = quant_arguments(&mut buffers, &input);- args.d_transform = MatmulDOps {+ args.output.ops = MatmulDOps {ab_scale: scale,bias: Some(&bias_buf),..MatmulDOps::none()@@ -362,7 +365,7 @@ fn parity_bf16_quant_rht(.expect("MatmulCpuKernel");let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("cpu command buffer");let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);- cpu_args.d_transform = MatmulDOps {+ cpu_args.output.ops = MatmulDOps {bias: with_bias.then_some(&cpu_bias),rht_factors: Some(&cpu_rht),..MatmulDOps::none()@@ -383,7 +386,7 @@ fn parity_bf16_quant_rht(.expect("MatmulMetalKernel");let mut command_buffer = context.create_command_buffer(None, None).unwrap();let mut args = quant_arguments(&mut buffers, &input);- args.d_transform = MatmulDOps {+ args.output.ops = MatmulDOps {bias: with_bias.then_some(&metal_bias),rht_factors: Some(&metal_rht),..MatmulDOps::none()@@ -434,7 +437,7 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {let mut command_buffer = context.create_command_buffer(None, None).unwrap();let mut args = quant_arguments(&mut buffers, &input);- args.d_transform = MatmulDOps {+ args.output.ops = MatmulDOps {accumulate: true,..MatmulDOps::none()};@@ -726,7 +729,7 @@ fn a8w_mxu_output_bias_parity_bf16(.expect("CPU matmul kernel");let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &reference_input);- cpu_arguments.d_transform = MatmulDOps {+ cpu_arguments.output.ops = MatmulDOps {rht_factors: cpu_output_hadamard_factors.as_ref(),..MatmulDOps::none()};@@ -753,7 +756,7 @@ fn a8w_mxu_output_bias_parity_bf16(.expect("Metal matmul kernel");let mut metal_command_buffer = context.create_command_buffer(None, None).unwrap();let mut metal_arguments = quant_arguments(&mut metal_buffers, &input);- metal_arguments.d_transform = MatmulDOps {+ metal_arguments.output.ops = MatmulDOps {bias: Some(&metal_output_bias),rht_factors: metal_output_hadamard_factors.as_ref(),..MatmulDOps::none()@@ -795,8 +798,7 @@ fn run_widened_f32<B: Backend>(b,b_leading_dimension: None,b_transpose: true,- d: &mut y,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(&mut y, MatmulDOps::none()),gather_indices: None::<&B::GlobalBuffer>,m: input.m,n: input.n,diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/trellis_gemm_test.rsnew file mode 100644index 000000000..1d77d6182--- /dev/null+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/trellis_gemm_test.rs@@ -0,0 +1,189 @@+#![cfg(backend = "metal")]++use half::bf16;+use rand::{RngExt, SeedableRng, rngs::SmallRng};+use rstest::rstest;+use uzu_engine_macros::uzu_test;++use crate::{+ backends::{+ common::{+ Backend, Context, Kernels,+ kernel::matmul::{+ MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, TrellisFormat,+ },+ },+ metal::{Metal, MetalContext},+ },+ data_type::DataType,+ tests::{+ helpers::{buffer_to_vec, create_buffer_with_data, submit_command_buffer},+ util::shared_metal_context,+ },+};++const V4_T8: TrellisFormat = TrellisFormat {+ vector_width: 4,+ transition_bits: 8,+ restart_columns: Some(64),+};+const V2_T6: TrellisFormat = TrellisFormat {+ vector_width: 2,+ transition_bits: 6,+ restart_columns: None,+};+const V2_T4: TrellisFormat = TrellisFormat {+ vector_width: 2,+ transition_bits: 4,+ restart_columns: None,+};++fn decode_state(+ format: TrellisFormat,+ row: &[u8],+ column: usize,+) -> u16 {+ let width = format.vector_width as usize;+ let (block, local) =+ format.restart_columns.map_or((0, column), |restart| (column / restart as usize, column % restart as usize));+ let block_bytes = format.restart_columns.map_or(0, |restart| row_bytes(format, restart as usize));+ let start_bit = block * block_bytes * 8 + local / width * format.transition_bits as usize;+ (start_bit..start_bit + 16).fold(0u32, |state, bit| (state << 1) | u32::from(row[bit / 8] >> (7 - bit % 8) & 1))+ as u16+}++fn row_bytes(+ format: TrellisFormat,+ columns: usize,+) -> usize {+ let block_columns = format.restart_columns.map_or(columns, |restart| restart as usize);+ let transitions = block_columns / format.vector_width as usize - 1;+ let bytes_per_block = (16 + transitions * format.transition_bits as usize).div_ceil(8);+ columns.div_ceil(block_columns) * bytes_per_block+}++fn decode_level(+ format: TrellisFormat,+ row: &[u8],+ column: usize,+) -> i32 {+ let state = u32::from(decode_state(format, row, column));+ let hash = state.wrapping_mul(0xCFCC_B83F).wrapping_add(0x584B_4AA3);+ let hash = (hash ^ hash >> 16).wrapping_mul(0x85EB_CA6B);+ let hash_byte = ((hash ^ hash >> 16) >> (8 * (column % format.vector_width as usize))) & 0xFF;+ let field_sum: u32 = (0..4).map(|field| (hash_byte >> (2 * field)) & 3).sum();+ (8 * field_sum + ((3 * (hash_byte & 15)) & 15)) as i32 - 54+}++type MetalBuffer = <Metal as Backend>::GlobalBuffer;+type MetalMatmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel;++const CODEBOOK: [f32; 5] = [0.0123, 0.3, -0.25, 0.125, -0.0625];+const PADDING: usize = 24;+const SENTINEL: u16 = 0x7F7F;+const RELATIVE_TOLERANCE: f32 = 1e-2;+// Cases are ordered as (m, n, k).+const CASES: [(usize, usize, usize); 7] =+ [(1, 80, 5120), (17, 80, 5120), (512, 80, 5120), (16, 128, 64), (32, 128, 64), (1, 6, 64), (2048, 2048, 128)];++fn run_projection(+ context: &MetalContext,+ format: TrellisFormat,+ dimensions: (usize, usize, usize),+) {+ let (m, n, k) = dimensions;+ let mut rng = SmallRng::seed_from_u64((m * k + n) as u64);+ let stored: Vec<u8> = (0..n * row_bytes(format, k)).map(|_| rng.random()).collect();+ let row_scales: Vec<f32> = (0..n).map(|_| rng.random_range(0.001..0.01)).collect();+ let mut values: Vec<i8> = (0..m * k).map(|_| rng.random_range(-127..=127)).collect();+ if m > 1 {+ // Exercise the zero-activation path in the epilogue.+ values[k..2 * k].fill(0);+ }+ let group_sums_host: Vec<f32> = values+ .chunks_exact(k)+ .flat_map(|row| {+ (0..4).map(|group| row.iter().skip(group).step_by(4).fold(0.0, |sum, &value| sum + f32::from(value)))+ })+ .collect();+ let activation_scales_host: Vec<f32> = (0..m).map(|_| rng.random_range(0.001..0.01)).collect();+ let activations = create_buffer_with_data::<Metal, i8>(context, &values);+ let group_sums = create_buffer_with_data::<Metal, f32>(context, &group_sums_host);+ let activation_scales = create_buffer_with_data::<Metal, f32>(context, &activation_scales_host);+ let codes = create_buffer_with_data::<Metal, u8>(context, &stored);+ let row_scales_buffer = create_buffer_with_data::<Metal, f32>(context, &row_scales);+ let codebook = create_buffer_with_data::<Metal, f32>(context, &CODEBOOK);+ let mut output = create_buffer_with_data::<Metal, u16>(context, &vec![SENTINEL; m * (n + PADDING)]);+ let arguments = MatmulArguments {+ a: MatmulA::Trellis {+ values: &activations,+ column_group_sums: &group_sums,+ scales: &activation_scales,+ },+ b: MatmulB::Trellis {+ codes: &codes,+ row_scales: &row_scales_buffer,+ codebook: &codebook,+ format,+ },+ b_leading_dimension: None,+ b_transpose: true,+ output: MatmulOutput {+ values: &mut output,+ row_stride: Some((n + PADDING) as u32),+ ops: MatmulDOps::none(),+ },+ gather_indices: None::<&MetalBuffer>,+ m: m as u32,+ n: n as u32,+ k: k as u32,+ };+ let mut matmul =+ <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16).unwrap();+ let mut command_buffer = context.create_command_buffer(None, None).unwrap();+ matmul.encode(arguments, &mut command_buffer).unwrap();+ submit_command_buffer(command_buffer);+ let decoded_levels: Vec<i32> = stored+ .chunks_exact(row_bytes(format, k))+ .flat_map(|codes| (0..k).map(|column| decode_level(format, codes, column)))+ .collect();+ let actual = buffer_to_vec::<Metal, u16>(&output);+ let stride = n + PADDING;+ for (index, &got) in actual.iter().enumerate() {+ let token = index / stride;+ let row = index % stride;+ if row < n {+ let row_values = &values[token * k..(token + 1) * k];+ let levels = &decoded_levels[row * k..(row + 1) * k];+ let level_dot: i32 = levels.iter().zip(row_values).map(|(&level, &value)| level * i32::from(value)).sum();+ let sums = &group_sums_host[token * 4..token * 4 + 4];+ // A non-dyadic offset exposes rounding differences in the GPU dot.+ let offsets_dot =+ (((sums[0] * CODEBOOK[1]) + sums[1] * CODEBOOK[2]) + sums[2] * CODEBOOK[3]) + sums[3] * CODEBOOK[4];+ let dot = (level_dot as f32).mul_add(CODEBOOK[0], offsets_dot);+ let want = bf16::from_f32(dot * row_scales[row] * activation_scales_host[token]).to_f32();+ let got = bf16::from_bits(got).to_f32();+ assert!(+ (got - want).abs() <= RELATIVE_TOLERANCE * want.abs(),+ "{format:?} ({m}, {n}, {k}) token {token} row {row}: got {got}, want {want}"+ );+ } else {+ assert_eq!(got, SENTINEL, "{format:?} M {m} N {n} K {k}, token {token}, row {row}");+ }+ }+}++#[rstest]+#[test_attr(uzu_test)]+#[case::v4_t8(V4_T8)]+#[case::v2_t6(V2_T6)]+#[case::v2_t4(V2_T4)]+fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {+ let context = shared_metal_context();+ if !context.supports_mxu {+ return;+ }+ for (m, n, k) in CASES {+ run_projection(&context, format, (m, n, k));+ }+}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex 466c1a2bf..563d2d8c2 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -19,6 +19,7 @@ fn shape(b_transpose: true,b_leading_dimension: None,b_prologue: GemmBPrologueKind::FullPrecision,+ b_is_trellis: false,b_bits: None,b_group_size: None,signed_codes: false,@@ -70,33 +71,18 @@ fn policy_boundaries_are_preserved() {assert_eq!(policy::mxu_fp_tile(m, n, k), expected);}- for (m, n, group_size, expected) in [- (16, 4096, 31, Tile64x64x16_Simdgroups2x2),- // Full M blocks keep the narrow tile.- (4, 4096, 32, Tile8x32x32_Simdgroups1x1),- (8, 4096, 32, Tile8x32x32_Simdgroups1x1),- (16, 4096, 32, Tile8x32x32_Simdgroups1x1),- (24, 4096, 32, Tile8x32x32_Simdgroups1x1),- // A partial trailing block takes the wide tile.- (9, 4096, 32, Tile32x32x32_Simdgroups2x2),- (15, 4096, 32, Tile32x32x32_Simdgroups2x2),- (31, 4096, 32, Tile32x32x32_Simdgroups2x2),- (64, 6143, 32, Tile32x32x32_Simdgroups2x2),- (64, 6144, 32, Tile64x64x32_Simdgroups2x2),+ for (m, n, group_size, family, expected) in [+ (16, 4096, 31, MTLGPUFamily::Apple7, Tile64x64x16_Simdgroups2x2),+ (16, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),+ (8, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),+ (9, 4096, 32, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),+ (9, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),+ (31, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),+ (32, 4096, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),+ (64, 6143, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),+ (64, 6144, 32, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),] {- assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, MTLGPUFamily::Apple7), expected);- }-- // Apple8 retains the older wide-tile policy; Apple9 and newer keep the narrow tile.- for apple_gpu_family in [MTLGPUFamily::Apple7, MTLGPUFamily::Apple8] {- for m in [9, 15, 31] {- assert_eq!(policy::simdgroup_quant_tile(m, 4096, 32, apple_gpu_family), Tile32x32x32_Simdgroups2x2);- }- }- for apple_gpu_family in [MTLGPUFamily::Apple9, MTLGPUFamily::Apple10] {- for m in [9, 15, 31] {- assert_eq!(policy::simdgroup_quant_tile(m, 4096, 32, apple_gpu_family), Tile8x32x32_Simdgroups1x1);- }+ assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, family), expected);}}@@ -163,6 +149,24 @@ fn selection_fallbacks_and_split_k_are_preserved() {assert_eq!(problem(zero, DataType::BF16).select_plan().split_k, 1);}+#[uzu_test]+fn trellis_plan_matches_projection_cases() {+ use GemmTiling::*;++ for (m, n, k, tiling, split_k) in [+ (16, 128, 64, Tile16x32x256_Simdgroups1x1, 1),+ (17, 80, 64, Tile64x64x256_Simdgroups2x2, 1),+ (2048, 2048, 128, Tile128x128x256_Simdgroups4x4, 1),+ (1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),+ ] {+ let mut trellis_shape = shape(m, n, k);+ trellis_shape.a_full_precision = false;+ trellis_shape.b_is_trellis = true;+ let plan = problem(trellis_shape, DataType::BF16).select_plan();+ assert_eq!((plan.engine, plan.tiling, plan.split_k), (GemmEngine::Mxu, tiling, split_k), "M {m} N {n} K {k}");+ }+}+#[uzu_test]fn forced_engine_errors_are_preserved() {let huge = shape(u32::MAX, u32::MAX, u32::MAX);@@ -176,7 +180,7 @@ fn forced_engine_errors_are_preserved() {invalid_layout.b_transpose = false;assert_eq!(problem(invalid_layout, DataType::BF16).select_plan_for_engine(GemmEngine::Mxu),- Err(GemmPlanError::UnsupportedQuantLayout)+ Err(GemmPlanError::UnsupportedLayout("quantized weights require transposed contiguous B")));}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rsindex 2c158982a..8e2264b52 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs@@ -119,6 +119,7 @@ fn quant_shape(b_transpose: true,b_leading_dimension: None,b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,+ b_is_trellis: false,b_bits: Some(4),b_group_size: Some(32),signed_codes: false,@@ -138,6 +139,7 @@ fn block_unaligned_quantized_k_stays_on_gemv() {b_transpose: true,b_leading_dimension: None,b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,+ b_is_trellis: false,b_bits: Some(4),b_group_size: Some(64),signed_codes: false,diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rsindex 5684be512..9121e6a80 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs@@ -1,5 +1,3 @@-use std::fmt::Write;-use metal::MTLGPUFamily;use uzu_engine_macros::uzu_test;use xxhash_rust::xxh3::xxh3_64;@@ -58,6 +56,7 @@ fn problem(b_transpose: true,b_leading_dimension: None,b_prologue: prologue,+ b_is_trellis: false,b_bits: Some(bits),b_group_size: Some(group),signed_codes: false,@@ -72,9 +71,6 @@ fn problem(fn table_is_complete_and_fingerprint_is_stable() {let mut canonical = Vec::new();let mut matched_rows = vec![false; ROWS.len()];- let mut tuned = 0;- let mut main_gemv = 0;- let mut main_gemm = 0;for &(device_label, device_name, gpu_core_count, apple_gpu_family, supports_mxu) in &DEVICES {for &(format_name, bits, group, prologue) in &FORMATS {for m in 2..=7 {@@ -96,11 +92,6 @@ fn table_is_complete_and_fingerprint_is_stable() {let (row_index, row) = matches[0];matched_rows[row_index] = true;let selected = row.route;- match selected {- QmvRoute::Tuned(_) => tuned += 1,- QmvRoute::MainGemv(_) => main_gemv += 1,- QmvRoute::MainGemm(_) => main_gemm += 1,- }canonical.push(format!("{device_label}|{format_name}|{shape_name}|{m}|{n}|{k}|{selected:?}"));let problem = problem(m, n, k, bits, group, prologue);assert_eq!(route(device_name, apple_gpu_family, supports_mxu, &problem, true), Some(selected));@@ -134,14 +125,7 @@ fn table_is_complete_and_fingerprint_is_stable() {assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");canonical.sort();assert_eq!(canonical.len(), 1176);- let mut expanded = String::new();- for line in canonical {- writeln!(&mut expanded, "{line}").expect("writing to a String must succeed");- }- assert_eq!(tuned, 864);- assert_eq!(main_gemv, 221);- assert_eq!(main_gemm, 91);- assert_eq!(xxh3_64(expanded.trim_end().as_bytes()), FROZEN_PLAN_FINGERPRINT);+ assert_eq!(xxh3_64(canonical.join("\n").as_bytes()), FROZEN_PLAN_FINGERPRINT);}#[uzu_test]@@ -229,23 +213,18 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {#[uzu_test]fn family_lookup_requires_one_unanimous_route() {let m1_route = problem(4, 8192, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- let measured_m1 = route("Apple M1", MTLGPUFamily::Apple7, false, &m1_route, true);- for device_name in ["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"] {- assert_eq!(route(device_name, MTLGPUFamily::Apple7, false, &m1_route, true), measured_m1);- }-let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);- let measured_m2 = route("Apple M2", MTLGPUFamily::Apple8, false, &unanimous, true);- for device_name in ["Apple M2 Max", "Apple M2 Ultra"] {- assert_eq!(route(device_name, MTLGPUFamily::Apple8, false, &unanimous, true), measured_m2);- }- let measured_m3_max = route("Apple M3 Max", MTLGPUFamily::Apple9, false, &unanimous, true);- for device_name in ["Apple M3", "Apple M3 Pro", "Apple M4 Max"] {- assert_eq!(route(device_name, MTLGPUFamily::Apple9, false, &unanimous, true), measured_m3_max);- }- let measured_m5_max = route("Apple M5 Max", MTLGPUFamily::Apple10, true, &unanimous, true);- for device_name in ["Apple M5", "Apple M5 Pro"] {- assert_eq!(route(device_name, MTLGPUFamily::Apple10, true, &unanimous, true), measured_m5_max);+ let families: [(&MatmulShape, MTLGPUFamily, bool, &str, &[&str]); 4] = [+ (&m1_route, MTLGPUFamily::Apple7, false, "Apple M1", &["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"]),+ (&unanimous, MTLGPUFamily::Apple8, false, "Apple M2", &["Apple M2 Max", "Apple M2 Ultra"]),+ (&unanimous, MTLGPUFamily::Apple9, false, "Apple M3 Max", &["Apple M3", "Apple M3 Pro", "Apple M4 Max"]),+ (&unanimous, MTLGPUFamily::Apple10, true, "Apple M5 Max", &["Apple M5", "Apple M5 Pro"]),+ ];+ for (shape, family, supports_mxu, measured_device, aliases) in families {+ let measured = route(measured_device, family, supports_mxu, shape, true);+ for device in aliases {+ assert_eq!(route(device, family, supports_mxu, shape, true), measured);+ }}let disagreement = problem(3, 5120, 6144, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);diff --git a/crates/uzu-engine/unit/common/matmul/harness.rs b/crates/uzu-engine/unit/common/matmul/harness.rsindex 84878e69f..c729c08f3 100644--- a/crates/uzu-engine/unit/common/matmul/harness.rs+++ b/crates/uzu-engine/unit/common/matmul/harness.rs@@ -10,7 +10,7 @@ use crate::{Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,kernel::{Kernels,- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},},},cpu::Cpu,@@ -170,8 +170,7 @@ fn run<B: Backend, T: ArrayElement + Float>(},b_leading_dimension: None,b_transpose: input.case.b_transpose,- d: &mut d_buffer,- d_transform,+ output: MatmulOutput::new(&mut d_buffer, d_transform),gather_indices: None,m,n,diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rsindex 482677f87..1149155d0 100644--- a/crates/uzu-engine/unit/common/matmul/quant.rs+++ b/crates/uzu-engine/unit/common/matmul/quant.rs@@ -17,8 +17,8 @@ use crate::{kernel::{ActivationQuantization, ActivationTransform, Kernels,matmul::{- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, QuantParams,- QuantParamsLayout, QuantizedB, QuantizedCorrection,+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,+ QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection,},},},@@ -445,8 +445,7 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(b,b_leading_dimension: None,b_transpose: true,- d: y,- d_transform: MatmulDOps::none(),+ output: MatmulOutput::new(y, MatmulDOps::none()),gather_indices: None,m: input.m,n: input.n,