Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Trellis projection for M5 (#888)

CCYeh663d27ffe9c9
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rs
index 305dad425..3f8897fa3 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/arguments.rs
@@ -1,4 +1,4 @@
-use super::{d_ops::MatmulDOps, matmul_a::MatmulA, matmul_b::MatmulB};
+use super::{MatmulOutput, matmul_a::MatmulA, matmul_b::MatmulB};
use crate::backends::common::{Backend, BufferMut, BufferRef};
pub struct MatmulArguments<
@@ -13,8 +13,7 @@ pub struct MatmulArguments<
pub b: MatmulB<TB>,
pub b_leading_dimension: Option<u32>,
pub b_transpose: bool,
- pub d: TD,
- pub d_transform: MatmulDOps<'d, B>,
+ pub output: MatmulOutput<'d, B, TD>,
pub gather_indices: Option<TI>,
pub m: u32,
pub n: u32,
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
index 808aaf441..24d59b226 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/error.rs
@@ -17,10 +17,16 @@ pub enum MatmulError<B: Backend> {
bit: GemmDTransform,
path: &'static str,
},
- #[error("Unsupported B layout on path {path}")]
+ #[error("Unsupported layout on path {path}")]
UnsupportedLayout {
path: &'static str,
},
+ #[error("Unsupported output row stride {stride} for {columns} columns on path {path}")]
+ UnsupportedOutputStride {
+ path: &'static str,
+ stride: u32,
+ columns: u32,
+ },
#[error("Incompatible A operand for {path}: {reason}")]
IncompatibleA {
path: &'static str,
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
index 8e0849676..4b3473809 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_a.rs
@@ -1,6 +1,6 @@
use std::mem::size_of;
-use crate::backends::common::{BufferRef, gpu_types::gemm::GemmAPrologueKind};
+use crate::backends::common::BufferRef;
#[derive(Clone, Copy, PartialEq)]
pub enum Int8CodeLayout {
@@ -58,17 +58,9 @@ pub enum MatmulA<T: BufferRef> {
scale_group_size: u32,
code_layout: Int8CodeLayout,
},
-}
-
-impl<T: BufferRef> MatmulA<T> {
- pub fn prologue_kind(&self) -> GemmAPrologueKind {
- match self {
- Self::FullPrecision {
- ..
- } => GemmAPrologueKind::FullPrecision,
- Self::Int8Symmetric {
- ..
- } => GemmAPrologueKind::Int8Symmetric,
- }
- }
+ Trellis {
+ values: T,
+ column_group_sums: T,
+ scales: T,
+ },
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
index 24bb89d24..8cb287064 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/matmul_b.rs
@@ -7,11 +7,24 @@ use crate::{
data_type::DataType,
};
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TrellisFormat {
+ pub vector_width: u32,
+ pub transition_bits: u32,
+ pub restart_columns: Option<u32>,
+}
+
pub enum MatmulB<TB: BufferRef> {
FullPrecision {
b: TB,
},
Quantized(QuantizedB<TB>),
+ Trellis {
+ codes: TB,
+ row_scales: TB,
+ codebook: TB,
+ format: TrellisFormat,
+ },
}
pub struct QuantizedB<TB: BufferRef> {
@@ -82,10 +95,17 @@ impl<TB: BufferRef> QuantizedB<TB> {
}
impl<TB: BufferRef> MatmulB<TB> {
+ pub fn is_trellis(&self) -> bool {
+ matches!(self, Self::Trellis { .. })
+ }
+
pub fn quantized(&self) -> Option<&QuantizedB<TB>> {
match self {
Self::FullPrecision {
..
+ }
+ | Self::Trellis {
+ ..
} => None,
Self::Quantized(quantized) => Some(quantized),
}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
index aab560dee..af2eef3ba 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/mod.rs
@@ -4,6 +4,7 @@ mod error;
mod kernel;
mod matmul_a;
mod matmul_b;
+mod output;
pub mod routing;
pub use arguments::MatmulArguments;
@@ -11,7 +12,8 @@ pub use d_ops::MatmulDOps;
pub use error::MatmulError;
pub use kernel::MatmulKernel;
pub use matmul_a::{Int8CodeLayout, MatmulA};
-pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection};
+pub use matmul_b::{MatmulB, QuantizedB, QuantizedCorrection, TrellisFormat};
+pub use output::MatmulOutput;
pub use routing::{ActivationFormat, MatmulShape};
use crate::backends::common::gpu_types::{QUANT_PARAMS_GROUP_OUTPUT_ALIGNMENT, QuantizationMode};
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs
new file mode 100644
index 000000000..c7cb76391
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/output.rs
@@ -0,0 +1,38 @@
+use super::{MatmulDOps, MatmulError};
+use crate::backends::common::{Backend, BufferMut};
+
+pub struct MatmulOutput<'d, B: Backend, T: BufferMut<Backend = B>> {
+ pub values: T,
+ /// Elements between output rows; `None` means the logical output width `n`.
+ pub row_stride: Option<u32>,
+ pub ops: MatmulDOps<'d, B>,
+}
+
+impl<'d, B: Backend, T: BufferMut<Backend = B>> MatmulOutput<'d, B, T> {
+ pub fn new(
+ values: T,
+ ops: MatmulDOps<'d, B>,
+ ) -> Self {
+ Self {
+ values,
+ row_stride: None,
+ ops,
+ }
+ }
+
+ pub(crate) fn into_contiguous(
+ self,
+ columns: u32,
+ path: &'static str,
+ ) -> Result<(T, MatmulDOps<'d, B>), MatmulError<B>> {
+ let stride = self.row_stride.unwrap_or(columns);
+ if stride != columns {
+ return Err(MatmulError::UnsupportedOutputStride {
+ path,
+ stride,
+ columns,
+ });
+ }
+ Ok((self.values, self.ops))
+ }
+}
diff --git a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs
index 658efa276..98d2df91d 100644
--- a/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/matmul/routing.rs
@@ -18,6 +18,7 @@ pub struct MatmulShape {
pub b_transpose: bool,
pub b_leading_dimension: Option<u32>,
pub b_prologue: GemmBPrologueKind,
+ pub b_is_trellis: bool,
pub b_bits: Option<u32>,
pub b_group_size: Option<u32>,
pub signed_codes: bool,
@@ -45,13 +46,14 @@ impl MatmulShape {
b_transpose: arguments.b_transpose,
b_leading_dimension: arguments.b_leading_dimension,
b_prologue: arguments.b.b_prologue(),
+ b_is_trellis: arguments.b.is_trellis(),
b_bits: arguments.b.bits_per_b(),
b_group_size: arguments.b.group_size(),
signed_codes: arguments.b.signed_codes(),
a_full_precision: matches!(arguments.a, MatmulA::FullPrecision { .. }),
gathered: arguments.gather_indices.is_some(),
params_layout: arguments.b.quant_params_layout(),
- d_transform: arguments.d_transform.mask(),
+ d_transform: arguments.output.ops.mask(),
}
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index 5273d789b..0e72889e0 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -67,24 +67,24 @@ impl MatmulKernel for MatmulCpuKernel {
>,
command_buffer: &mut CpuCommandBufferEncoding,
) -> Result<(), CpuError> {
- let output_scale = arguments.d_transform.ab_scale;
- let accumulate = arguments.d_transform.accumulate;
- let bias_buffer = arguments.d_transform.bias;
- let post_rht = arguments.d_transform.rht_factors;
- let soft_cap = arguments.d_transform.soft_cap;
-
let MatmulArguments {
a,
b,
b_leading_dimension,
b_transpose,
- mut d,
+ output,
m,
n,
k,
gather_indices,
..
} = arguments;
+ let (mut d, ops) = output.into_contiguous(n, "CpuMatmul")?;
+ let output_scale = ops.ab_scale;
+ let accumulate = ops.accumulate;
+ let bias_buffer = ops.bias;
+ let post_rht = ops.rht_factors;
+ let soft_cap = ops.soft_cap;
let m_u = m as usize;
let n_u = n as usize;
@@ -146,6 +146,14 @@ impl MatmulKernel for MatmulCpuKernel {
code_layout,
}
},
+ MatmulA::Trellis {
+ ..
+ } => {
+ return Err(MatmulError::UnsupportedLayout {
+ path: "CpuMatmul",
+ }
+ .into());
+ },
};
let bias_ptr = bias_buffer.map(|bias| SendPtr(bias.cpu_ptr().as_ptr().cast::<u8>().cast_const()));
let gather_ptr = gather_indices.map(|indices| {
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
index e6674ab1f..9ca424454 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
@@ -57,6 +57,11 @@ impl WeightData {
transpose: b_transpose,
})
},
+ MatmulB::Trellis {
+ ..
+ } => Err(MatmulError::UnsupportedLayout {
+ path: "CpuMatmul",
+ }),
MatmulB::Quantized(quantized) => Ok(WeightData::Quantized {
weights: buffer_ptr(quantized.codes),
scales: buffer_ptr(quantized.scales),
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
index dec62f708..bf19f9ea0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/fallback.rs
@@ -6,7 +6,7 @@ use crate::{
Backend, BufferRef, CommandBufferEncoding, Kernels,
kernel::{
AttentionArguments, AttentionKernelConfig, SoftmaxKernel,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},
},
},
metal::{
@@ -119,11 +119,13 @@ impl AttentionFallback {
},
b_leading_dimension: Some(self.num_groups * self.head_dim),
b_transpose: true,
- d: &mut group_scores,
- d_transform: MatmulDOps {
- ab_scale: scale,
- ..MatmulDOps::none()
- },
+ output: MatmulOutput::new(
+ &mut group_scores,
+ MatmulDOps {
+ ab_scale: scale,
+ ..MatmulDOps::none()
+ },
+ ),
gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,
m: gqa_factor * suffix_length,
n: sequence_length,
@@ -168,8 +170,7 @@ impl AttentionFallback {
},
b_leading_dimension: Some(self.num_groups * self.head_dim),
b_transpose: false,
- d: &mut group_output,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(&mut group_output, MatmulDOps::none()),
gather_indices: None::<&<Metal as Backend>::ScratchBuffer>,
m: gqa_factor * suffix_length,
n: self.head_dim,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 301c78864..beee31ab8 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -109,15 +109,15 @@ struct MxuMmaCore {
? SIMDGROUP_BLOCK_N
: short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));
- schedules::TileContext tile_context;
- tile_context.simdgroup_limit_m = simdgroup_limit_m;
- tile_context.simdgroup_limit_n = simdgroup_limit_n;
- tile_context.block_col = block_col;
- tile_context.tile_col_offset = tile_col_offset;
- tile_context.tile_block_cols =
- ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start)));
- tile_context.k_offset = k_offset;
- tile_context.abs_row_base = uint(geometry.block_row_start) + tile_row_offset;
+ const schedules::TileContext tile_context{
+ .simdgroup_limit_m = simdgroup_limit_m,
+ .simdgroup_limit_n = simdgroup_limit_n,
+ .block_col = block_col,
+ .tile_col_offset = tile_col_offset,
+ .tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start))),
+ .k_offset = k_offset,
+ .abs_row_base = uint(geometry.block_row_start) + tile_row_offset,
+ };
const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);
const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h
new file mode 100644
index 000000000..443f847e6
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/trellis.h
@@ -0,0 +1,140 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../../common/defines.h"
+#include "../../../../generated/gemm.h"
+#include "cursor.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+namespace trellis {
+
+struct GemmTrellisFormat {
+ uint32_t vector_width;
+ uint32_t transition_bits;
+ uint32_t restart_columns;
+};
+
+UZU_CONST uint K_STEP = 64, WEIGHT_BIAS = 54;
+UZU_CONST uint STATE_BITS = 16, BYTE_BITS = 8, V4_VECTOR_WIDTH = 4, V2T6_TRANSITION_BITS = 6;
+
+UZU_CONST uint HASH_MULTIPLIER = 0xCFCCB83Fu, HASH_INCREMENT = 0x584B4AA3u;
+UZU_CONST uint HASH_AVALANCHE_MULTIPLIER = 0x85EBCA6Bu, HASH_FOLD_BITS = 16;
+
+UZU_CONST uint TWO_BIT_FIELD_SHIFT = 2, NIBBLE_SHIFT = 4, FIELD_SUM_SHIFT = 3;
+UZU_CONST uint TWO_BIT_FIELDS_MASK = 0x33333333u, BYTE_NIBBLES_MASK = 0x0F0F0F0Fu;
+UZU_CONST uint FIELD_PAIR_SUM_MULTIPLIER = 3, DITHER_MULTIPLIER = 3;
+
+// Row layout
+static METAL_FUNC uint block_bytes(const GemmTrellisFormat format, const uint block_columns) {
+ const uint state_count = block_columns / format.vector_width;
+ const uint transition_count = state_count - 1;
+ return (STATE_BITS + transition_count * format.transition_bits + BYTE_BITS - 1) / BYTE_BITS;
+}
+
+static METAL_FUNC uint row_bytes(const GemmTrellisFormat format, const uint code_columns) {
+ const uint block_columns = format.restart_columns == 0 ? code_columns : format.restart_columns;
+ return code_columns / block_columns * block_bytes(format, block_columns);
+}
+
+// MSB-first codes. code_column must be a multiple of 4.
+static METAL_FUNC uint2
+states_at(const GemmTrellisFormat format, const device uchar* code_row, const uint code_column) {
+ if (format.vector_width == V4_VECTOR_WIDTH) {
+ const device uchar* block_codes = code_row;
+ uint block_column = code_column;
+ if (format.restart_columns != 0) {
+ block_codes += code_column / format.restart_columns * block_bytes(format, format.restart_columns);
+ block_column %= format.restart_columns;
+ }
+ const device uchar* state_bytes = block_codes + block_column / V4_VECTOR_WIDTH;
+ return uint2(uint(state_bytes[0]) << BYTE_BITS | state_bytes[1], 0);
+ }
+ const uint first_bit = code_column / format.vector_width * format.transition_bits;
+ const device uchar* bytes = code_row + first_bit / BYTE_BITS;
+ const uint shift = first_bit % BYTE_BITS;
+ const uint fourth_byte = format.transition_bits == V2T6_TRANSITION_BITS ? uint(bytes[3]) : 0;
+ const uint window = uint(bytes[0]) << 24 | uint(bytes[1]) << 16 | uint(bytes[2]) << 8 | fourth_byte;
+ const uint state_mask = (1u << STATE_BITS) - 1;
+ const uint first_state = (window >> (16 - shift)) & state_mask;
+ const uint second_state = (window >> (16 - format.transition_bits - shift)) & state_mask;
+ return uint2(first_state, second_state);
+}
+
+// State -> hash -> biased weights
+static METAL_FUNC uint hash_without_final_fold(const uint code_state) {
+ uint hash = code_state * HASH_MULTIPLIER + HASH_INCREMENT;
+ hash ^= hash >> HASH_FOLD_BITS;
+ return hash * HASH_AVALANCHE_MULTIPLIER;
+}
+
+static METAL_FUNC uint final_fold(const uint hash) { return hash ^ (hash >> HASH_FOLD_BITS); }
+
+// Per byte b: weight + WEIGHT_BIAS = 8 * (sum of b's four 2-bit fields) + ((3 * (b & 15)) & 15).
+static METAL_FUNC uint biased_weights(const uint hash_bytes) {
+ const uint nibble_field_sums =
+ hash_bytes - FIELD_PAIR_SUM_MULTIPLIER * ((hash_bytes >> TWO_BIT_FIELD_SHIFT) & TWO_BIT_FIELDS_MASK);
+ const uint byte_field_sums = (nibble_field_sums + (nibble_field_sums >> NIBBLE_SHIFT)) & BYTE_NIBBLES_MASK;
+ const uint low_nibble_dither = ((hash_bytes & BYTE_NIBBLES_MASK) * DITHER_MULTIPLIER) & BYTE_NIBBLES_MASK;
+ return (byte_field_sums << FIELD_SUM_SHIFT) + low_nibble_dither;
+}
+
+static METAL_FUNC uint
+decode_biased_weights(const GemmTrellisFormat format, const device uchar* code_row, const uint code_column) {
+ const uint2 code_states = states_at(format, code_row, code_column);
+ const uint first_hash = hash_without_final_fold(code_states.x);
+ if (format.vector_width == V4_VECTOR_WIDTH) {
+ return biased_weights(final_fold(first_hash));
+ }
+ const uint second_hash = hash_without_final_fold(code_states.y);
+ const uint low_halves = insert_bits(first_hash, second_hash, HASH_FOLD_BITS, HASH_FOLD_BITS);
+ const uint high_halves = insert_bits(second_hash, first_hash >> HASH_FOLD_BITS, 0, HASH_FOLD_BITS);
+ return biased_weights(low_halves ^ high_halves);
+}
+
+} // namespace trellis
+
+namespace quantized {
+
+template <typename Fragment, bool ALIGNED>
+struct TrellisCursor {
+ using Ops = typename Fragment::FragmentOpsType;
+ UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);
+
+ const device uint8_t* code_rows;
+ uint code_row_stride_bytes;
+ short tile_row_limit;
+ short2 position;
+ uint k_offset;
+ trellis::GemmTrellisFormat format;
+
+ METAL_FUNC Fragment load(const uint) const thread {
+ const short row_limit = tile_row_limit - position.y;
+ Fragment tile;
+ for_each_fragment_row<Fragment>([&](ushort fragment_row, ushort row_slot, short row_offset) {
+ const bool live = ALIGNED || row_offset < row_limit;
+ const device uint8_t* code_row = code_rows + int(row_offset) * int(code_row_stride_bytes);
+ METAL_PRAGMA_UNROLL
+ for (ushort k_fragment = 0; k_fragment < Fragment::COL_FRAGMENTS; ++k_fragment) {
+ reinterpret_cast<thread uint*>(&tile.fragment_at(fragment_row, k_fragment))[row_slot] =
+ live ? trellis::decode_biased_weights(
+ format,
+ code_row,
+ k_offset + uint(k_fragment * Ops::FRAGMENT_ROWS) + uint(position.x)
+ )
+ : 0u;
+ }
+ });
+ return tile;
+ }
+
+ METAL_FUNC void advance() thread { k_offset += uint(BLOCK_K); }
+};
+
+} // namespace quantized
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
new file mode 100644
index 000000000..137171472
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
@@ -0,0 +1,194 @@
+#include "../../common/dsl.h"
+#include "../../common/defines.h"
+#include "../../common/thread_context.h"
+#include "../generated/gemm.h"
+
+#include "common/gemm_tiling.h"
+#include "common/mxu_mma_core.h"
+#include "common/quantized/trellis.h"
+
+using namespace metal;
+using namespace uzu::gemm;
+
+#define REDUCE_THREADS 256
+#define REDUCE_VECTOR_WIDTH 4
+
+namespace {
+
+struct TrellisEpilogue {
+ const device float* scale_and_offsets;
+ const device float4* group_sums;
+ const device float* activation_scales;
+ const device float* row_scales;
+ device bfloat* output;
+ uint output_row_stride;
+
+ METAL_FUNC void store(const int biased_dot, const uint token, const uint row) const {
+ const float4 group_sum = group_sums[token];
+ const int level_dot =
+ biased_dot - int(trellis::WEIGHT_BIAS) * int(group_sum.x + group_sum.y + group_sum.z + group_sum.w);
+ const float activation_scale = activation_scales[token];
+ const float4 offsets =
+ float4(scale_and_offsets[1], scale_and_offsets[2], scale_and_offsets[3], scale_and_offsets[4]);
+ const float dot = float(level_dot) * scale_and_offsets[0] + metal::dot(group_sum, offsets);
+ output[token * output_row_stride + row] = bfloat(dot * row_scales[row] * activation_scale);
+ }
+};
+
+} // namespace
+
+template <GemmTiling GEMM_TILING>
+VARIANTS(
+ GEMM_TILING,
+ GemmTiling::Tile16x32x256_Simdgroups1x1,
+ GemmTiling::Tile64x64x256_Simdgroups2x2,
+ GemmTiling::Tile128x128x256_Simdgroups4x4)
+KERNEL(GemmTrellis)(
+ const device int8_t* activations,
+ const device float4* column_group_sums,
+ const device float* activation_scales,
+ const device uint8_t* codes,
+ const device float* row_scales,
+ const device float* scale_and_offsets,
+ device bfloat* output,
+ const constant uzu::matmul::GemmParams* params,
+ const constant uint& group_count_x,
+ const constant uint& group_count_y,
+ const constant uint& group_count_z,
+ const GemmAlignment alignment SPECIALIZE,
+ const uint trellis_vector_width SPECIALIZE,
+ const uint trellis_transition_bits SPECIALIZE,
+ const uint trellis_restart_columns SPECIALIZE,
+ const uint group_x GROUPS(group_count_x),
+ const uint group_y GROUPS(group_count_y),
+ const uint group_z GROUPS(group_count_z),
+ const uint thread_x THREADS(METAL_SIMD_SIZE),
+ const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),
+ const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),
+ const ThreadContext thread_context
+) {
+ (void)thread_x;
+ (void)thread_y;
+ (void)thread_z;
+
+ using LeftOperand =
+ operands::LeftOperandFor<GemmAPrologueKind::Int8Symmetric, bfloat, ushort(trellis::K_STEP), false>;
+ // A stock int8 right operand, only for the tile constants of `Core`.
+ using RightOperand =
+ operands::RightOperandFor<GemmBPrologueKind::ScaleSymmetricDequant, ushort(8), ushort(trellis::K_STEP), bfloat>;
+ using Core = MxuMmaCore<bfloat, GEMM_TILING, true, LeftOperand, RightOperand>;
+ using Ops = typename Core::FragmentOps;
+ using Products = uzu::matmul::Fragment<int, Core::TILES_M, Core::TILES_N, Ops>;
+ using RightFragment = uzu::matmul::Fragment<int8_t, Core::TILES_N, Core::TILES_K, Ops, uzu::matmul::ReadDirect, true>;
+ const trellis::GemmTrellisFormat trellis_format{
+ trellis_vector_width,
+ trellis_transition_bits,
+ trellis_restart_columns
+ };
+
+ const uint simdgroup = thread_context.simdgroup_index;
+ const uint token_base =
+ group_y * Core::THREADGROUP_BLOCK_M + Core::SIMDGROUP_BLOCK_M * (simdgroup / Core::SIMDGROUPS_PER_COLUMN);
+ const uint row_base =
+ group_x * Core::THREADGROUP_BLOCK_N + Core::SIMDGROUP_BLOCK_N * (simdgroup % Core::SIMDGROUPS_PER_COLUMN);
+ const short token_end = short(min(int(Core::SIMDGROUP_BLOCK_M), int(params->M) - int(token_base)));
+ const short row_end = short(min(int(Core::SIMDGROUP_BLOCK_N), int(params->N) - int(row_base)));
+ const schedules::TileContext tile_context{
+ .simdgroup_limit_m = token_end,
+ .simdgroup_limit_n = row_end,
+ .k_offset = group_z * params->aligned_inner_iterations * trellis::K_STEP,
+ .abs_row_base = token_base,
+ };
+
+ const uint code_row_bytes = trellis::row_bytes(trellis_format, params->K);
+ const short2 position = Ops::get_position(thread_context.simd_lane_id);
+ const auto left_storage = operands::pack_left<LeftOperand, bfloat>(nullptr, activations, nullptr, nullptr);
+
+ Products products;
+ products.clear();
+ uzu::dispatch_bool(alignment.contains(GemmAlignment::M) || token_end == Core::SIMDGROUP_BLOCK_M, [&](auto aligned_m) {
+ uzu::dispatch_bool(alignment.contains(GemmAlignment::N) || row_end == Core::SIMDGROUP_BLOCK_N, [&](auto aligned_n) {
+ auto left = quantized::make_left_cursor<true, Core, LeftOperand, aligned_m.value>(
+ left_storage,
+ params,
+ tile_context,
+ thread_context
+ );
+ quantized::TrellisCursor<RightFragment, aligned_n.value> trellis_cursor{
+ codes + size_t(row_base + position.y) * code_row_bytes,
+ code_row_bytes,
+ row_end,
+ position,
+ tile_context.k_offset,
+ trellis_format
+ };
+ const int k_chunks = token_end > 0 && row_end > 0
+ ? int(params->aligned_inner_iterations) * int(trellis::K_STEP / Core::SIMDGROUP_BLOCK_K)
+ : 0;
+ METAL_PRAGMA_NO_UNROLL
+ for (int chunk = 0; chunk < k_chunks; ++chunk) {
+ auto left_tile = left.load(0u);
+ auto right_tile = trellis_cursor.load(0u);
+ uzu::matmul::fragment_mma(products, left_tile, right_tile);
+ left.advance();
+ trellis_cursor.advance();
+ }
+ });
+ });
+
+ const bool write_split_k_partials = params->aligned_inner_iterations * trellis::K_STEP < params->K;
+ const size_t partial_sum_offset = size_t(group_z) * params->M * params->N;
+ const TrellisEpilogue epilogue{
+ scale_and_offsets,
+ column_group_sums,
+ activation_scales,
+ row_scales,
+ output,
+ params->leading_dimension_d
+ };
+ products.map_coords(thread_context.simd_lane_id, [&](short token_offset, short row_offset, int value) {
+ if (token_offset < token_end && row_offset < row_end) {
+ const uint token = token_base + uint(token_offset);
+ const uint row = row_base + uint(row_offset);
+ if (write_split_k_partials) {
+ device int* partial_sums = reinterpret_cast<device int*>(output) + partial_sum_offset;
+ partial_sums[size_t(token) * params->N + row] = value;
+ } else {
+ epilogue.store(value, token, row);
+ }
+ }
+ return value;
+ });
+}
+
+KERNEL(GemmTrellisReduce)(
+ const device int4* partial_sums,
+ const device float4* column_group_sums,
+ const device float* activation_scales,
+ const device float* row_scales,
+ const device float* scale_and_offsets,
+ device bfloat* output,
+ const constant uint& token_count,
+ const constant uint& row_count,
+ const constant uint& partition_count,
+ const constant uint& output_stride,
+ const uint threadgroup_index GROUPS((token_count * (row_count / REDUCE_VECTOR_WIDTH)).div_ceil(REDUCE_THREADS)),
+ const uint thread_index_in_threadgroup THREADS(REDUCE_THREADS)
+) {
+ const uint row_groups = row_count / REDUCE_VECTOR_WIDTH;
+ const uint token_row_group = threadgroup_index * REDUCE_THREADS + thread_index_in_threadgroup;
+ if (token_row_group >= token_count * row_groups) {
+ return;
+ }
+ int4 partial_sum = int4(0);
+ for (uint partition = 0u; partition < partition_count; ++partition) {
+ partial_sum += partial_sums[partition * token_count * row_groups + token_row_group];
+ }
+ const uint token = token_row_group / row_groups;
+ const uint first_row = token_row_group % row_groups * REDUCE_VECTOR_WIDTH;
+ const TrellisEpilogue
+ epilogue{scale_and_offsets, column_group_sums, activation_scales, row_scales, output, output_stride};
+ for (uint index = 0u; index < REDUCE_VECTOR_WIDTH; ++index) {
+ epilogue.store(partial_sum[index], token, first_row + index);
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 0854876ad..fe4ae2bdf 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -5,6 +5,7 @@ use super::{
GemmEngine, GemmPlan,
selection::{GemmProblem, outer_block_k},
specialization::GemmSpecialization,
+ trellis::TrellisGemm,
};
use crate::{
backends::{
@@ -15,7 +16,7 @@ use crate::{
gemm::{GemmAPrologueKind, GemmAlignment, GemmBPrologueKind, GemmDTransform},
},
kernel::matmul::{
- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulError, MatmulShape,
+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulError, MatmulOutput, MatmulShape,
QuantParamsLayout, QuantParamsStrides, QuantizedB,
},
},
@@ -34,6 +35,7 @@ pub struct GemmKernel {
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
+ trellis: TrellisGemm,
kernels: HashMap<GemmSpecialization, GemmMetalKernel>,
split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,
}
@@ -48,6 +50,7 @@ impl GemmKernel {
weights_data_type,
input_data_type,
output_data_type,
+ trellis: TrellisGemm::default(),
kernels: HashMap::new(),
split_k_reduce: HashMap::new(),
}
@@ -138,12 +141,22 @@ impl GemmKernel {
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<(), MetalError> {
let shape = MatmulShape::from_arguments(&arguments);
+ // TODO: remove after GatherGEMM is supported
+ if arguments.gather_indices.is_some() {
+ return Err(MetalError::KernelDispatchFailed(
+ format!(
+ "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",
+ arguments.m, arguments.n
+ )
+ .into(),
+ ));
+ }
self.problem(shape, command_buffer.context())
.validate_engine(plan.engine)
.map_err(|error| MetalError::KernelDispatchFailed(Box::new(error)))?;
if !matches!(arguments.b, MatmulB::FullPrecision { .. })
- && arguments.d_transform.mask().contains(GemmDTransform::ACCUMULATE)
+ && arguments.output.ops.mask().contains(GemmDTransform::ACCUMULATE)
{
return Err(MatmulError::UnsupportedDOp {
bit: GemmDTransform::ACCUMULATE,
@@ -152,28 +165,37 @@ impl GemmKernel {
.into());
}
- let MatmulArguments {
- a,
- b,
- d,
- d_transform,
- ..
- } = arguments;
- match b {
+ match arguments.b {
MatmulB::FullPrecision {
b: weights,
- } => self.encode_weights(a, weights, None, d, d_transform, shape, plan, output_work, command_buffer),
+ } => self.encode_weights(
+ arguments.a,
+ weights,
+ None,
+ arguments.output,
+ shape,
+ plan,
+ output_work,
+ command_buffer,
+ ),
MatmulB::Quantized(quantized) => self.encode_weights(
- a,
+ arguments.a,
quantized.codes,
Some(quantized),
- d,
- d_transform,
+ arguments.output,
shape,
plan,
output_work,
command_buffer,
),
+ MatmulB::Trellis {
+ ..
+ } => {
+ if self.output_data_type != DataType::BF16 {
+ return Err(MatmulError::UnsupportedDataType(self.output_data_type).into());
+ }
+ self.trellis.encode_plan(arguments, plan, command_buffer)
+ },
}
}
@@ -182,14 +204,14 @@ impl GemmKernel {
a: MatmulA<impl BufferRef<Backend = Metal>>,
weights: WB,
quantized: Option<QuantizedB<WB>>,
- mut d: impl BufferMut<Backend = Metal>,
- d_transform: MatmulDOps<'_, Metal>,
+ output: MatmulOutput<'_, Metal, impl BufferMut<Backend = Metal>>,
shape: MatmulShape,
plan: GemmPlan,
output_work: &MatmulOutputWork,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<(), MetalError> {
let (m, n, k) = (shape.m, shape.n, shape.k);
+ let (mut d, d_transform) = output.into_contiguous(n, "Gemm")?;
let ab_scale = d_transform.ab_scale;
let output_bias = d_transform.bias;
let rht_factors = d_transform.rht_factors;
@@ -221,12 +243,11 @@ impl GemmKernel {
quantized.zero_point_strides(),
),
};
- let a_prologue = a.prologue_kind();
- let (a_full_precision, a_int8, a_scales, a_group_sums, a_group_size) = match &a {
+ let (a_full_precision, a_int8, a_scales, a_group_sums, a_prologue, a_group_size) = match &a {
MatmulA::FullPrecision {
values,
offset,
- } => (Some(values.subrange(*offset..)), None, None, None, None),
+ } => (Some(values.subrange(*offset..)), None, None, None, GemmAPrologueKind::FullPrecision, None),
MatmulA::Int8Symmetric {
values,
scales: activation_scales,
@@ -248,7 +269,22 @@ impl GemmKernel {
}
.into());
}
- (None, Some(*values), Some(*activation_scales), *activation_group_sums, Some(*scale_group_size))
+ (
+ None,
+ Some(*values),
+ Some(*activation_scales),
+ *activation_group_sums,
+ GemmAPrologueKind::Int8Symmetric,
+ Some(*scale_group_size),
+ )
+ },
+ MatmulA::Trellis {
+ ..
+ } => {
+ return Err(MatmulError::UnsupportedLayout {
+ path: "Gemm",
+ }
+ .into());
},
};
@@ -368,6 +404,14 @@ impl GemmKernel {
} => {
(None, Some(values), Some(scales), group_sums, GemmAPrologueKind::Int8Symmetric, Some(scale_group_size))
},
+ MatmulA::Trellis {
+ ..
+ } => {
+ return Err(MatmulError::UnsupportedLayout {
+ path: "Gemm",
+ }
+ .into());
+ },
};
let tiling = plan.tiling;
let split_k = plan.split_k;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
index 2350fd612..d45b0d0c0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -3,6 +3,7 @@ mod kernel;
mod policy;
mod selection;
mod specialization;
+mod trellis;
pub use error::GemmSpecializationError;
pub use kernel::GemmKernel;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index 7a439b16e..9e5ba1d04 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -27,9 +27,9 @@ const SIMDGROUP_QUANT_LARGE_M_MIN: u32 = 64;
const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;
const SPLIT_K_TARGET_TILES_FP: u32 = 512;
-const SPLIT_K_TARGET_TILES_A8: u32 = 256;
-const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;
-const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;
+pub(super) const SPLIT_K_TARGET_TILES_A8: u32 = 256;
+pub(super) const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;
+pub(super) const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;
const SPLIT_K_TARGET_TILES_A8_TILE32_W8: u32 = 1024;
fn bucket(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
index 6e5c0c8ae..b5c49acd9 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/selection.rs
@@ -10,6 +10,8 @@ use crate::{
data_type::DataType,
};
+pub(super) const TRELLIS_K_STEP: u32 = 64;
+
#[derive(Clone, Copy)]
pub struct GemmProblem {
shape: MatmulShape,
@@ -23,8 +25,8 @@ pub struct GemmProblem {
pub(super) enum GemmPlanError {
#[error("MXU engine is not available for this GEMM")]
MxuUnavailable,
- #[error("quantized GEMM requires transposed contiguous B")]
- UnsupportedQuantLayout,
+ #[error("GEMM input layout is unsupported: {0}")]
+ UnsupportedLayout(&'static str),
}
impl GemmProblem {
@@ -45,6 +47,9 @@ impl GemmProblem {
}
pub fn select_plan(self) -> GemmPlan {
+ if self.shape.b_is_trellis {
+ return select_trellis_plan(self.shape.m, self.shape.n, self.shape.k);
+ }
let engine = if self.supports_mxu && mxu_is_eligible(self.shape) {
GemmEngine::Mxu
} else {
@@ -58,6 +63,11 @@ impl GemmProblem {
self,
engine: GemmEngine,
) -> Result<GemmPlan, GemmPlanError> {
+ if self.shape.b_is_trellis {
+ let mut plan = select_trellis_plan(self.shape.m, self.shape.n, self.shape.k);
+ plan.engine = engine;
+ return Ok(plan);
+ }
self.validate_engine(engine)?;
Ok(self.finish_plan(engine, select_tiling(self.shape, engine, self.apple_gpu_family)))
}
@@ -70,7 +80,10 @@ impl GemmProblem {
return Err(GemmPlanError::MxuUnavailable);
}
if self.shape.is_quant() && (!self.shape.b_transpose || self.shape.b_leading_dimension.is_some()) {
- return Err(GemmPlanError::UnsupportedQuantLayout);
+ return Err(GemmPlanError::UnsupportedLayout("quantized weights require transposed contiguous B"));
+ }
+ if self.shape.b_is_trellis && !self.shape.k.is_multiple_of(TRELLIS_K_STEP) {
+ return Err(GemmPlanError::UnsupportedLayout("Trellis K must be divisible by 64"));
}
Ok(())
}
@@ -140,6 +153,36 @@ impl GemmProblem {
}
}
+pub(super) fn select_trellis_plan(
+ m: u32,
+ n: u32,
+ k: u32,
+) -> GemmPlan {
+ let (tiling, target_workgroups) = match policy::mxu_mn_tile(true, m, n) {
+ GemmTiling::Tile16x32x256_Simdgroups1x1 => {
+ (GemmTiling::Tile16x32x256_Simdgroups1x1, policy::SPLIT_K_TARGET_TILES_A8_TILE16X32)
+ },
+ GemmTiling::Tile32x64x256_Simdgroups2x2 => {
+ (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8_TILE32_W4)
+ },
+ GemmTiling::Tile128x128x256_Simdgroups4x4 => {
+ (GemmTiling::Tile128x128x256_Simdgroups4x4, policy::SPLIT_K_TARGET_TILES_A8)
+ },
+ _ => (GemmTiling::Tile64x64x256_Simdgroups2x2, policy::SPLIT_K_TARGET_TILES_A8),
+ };
+ let output_workgroups = (n.div_ceil(tiling.block_n()) * m.div_ceil(tiling.block_m())).max(1);
+ let k_steps = (k / TRELLIS_K_STEP).max(1);
+ let mut split_k = (target_workgroups / output_workgroups).clamp(1, k_steps);
+ while !k_steps.is_multiple_of(split_k) {
+ split_k -= 1;
+ }
+ GemmPlan {
+ engine: GemmEngine::Mxu,
+ tiling,
+ split_k,
+ }
+}
+
pub(super) fn outer_block_k(
shape: MatmulShape,
engine: GemmEngine,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
new file mode 100644
index 000000000..0929b24b3
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
@@ -0,0 +1,145 @@
+use std::collections::{HashMap, hash_map::Entry};
+
+use super::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP};
+use crate::backends::{
+ common::{
+ BufferMut, BufferRef, CommandBufferEncoding,
+ gpu_types::{
+ GemmParams,
+ gemm::{GemmAlignment, GemmTiling},
+ },
+ kernel::matmul::{MatmulA, MatmulArguments, MatmulB, MatmulError, TrellisFormat},
+ },
+ metal::{
+ Metal,
+ command_buffer::MetalCommandBufferEncoding,
+ error::MetalError,
+ kernel::{GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel},
+ },
+};
+
+#[derive(Default)]
+pub(super) struct TrellisGemm {
+ kernels: HashMap<(GemmTiling, TrellisFormat, GemmAlignment), GemmTrellisMetalKernel>,
+ reduce_kernel: Option<GemmTrellisReduceMetalKernel>,
+}
+
+impl TrellisGemm {
+ pub(super) fn encode_plan(
+ &mut self,
+ arguments: MatmulArguments<
+ '_,
+ Metal,
+ impl BufferRef<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
+ impl BufferMut<Backend = Metal>,
+ impl BufferRef<Backend = Metal>,
+ >,
+ plan: GemmPlan,
+ command_buffer: &mut MetalCommandBufferEncoding,
+ ) -> Result<(), MetalError> {
+ let unsupported = || {
+ MetalError::from(MatmulError::UnsupportedLayout {
+ path: "GemmTrellis",
+ })
+ };
+ let (
+ MatmulA::Trellis {
+ values: activations,
+ column_group_sums,
+ scales: activation_scales,
+ },
+ MatmulB::Trellis {
+ codes,
+ row_scales,
+ codebook,
+ format,
+ },
+ ) = (arguments.a, arguments.b)
+ else {
+ return Err(unsupported());
+ };
+ let (m, n, k) = (arguments.m, arguments.n, arguments.k);
+ let output_stride = arguments.output.row_stride.unwrap_or(n);
+ let (tiling, split_k) = (plan.tiling, plan.split_k);
+ if plan.engine != GemmEngine::Mxu
+ || !arguments.output.ops.mask().is_empty()
+ || !arguments.b_transpose
+ || arguments.b_leading_dimension.is_some()
+ || output_stride < n
+ || (split_k > 1 && !n.is_multiple_of(4))
+ {
+ return Err(unsupported());
+ }
+
+ let params = GemmParams {
+ M: m,
+ N: n,
+ K: k,
+ leading_dimension_a: k,
+ leading_dimension_d: output_stride,
+ threadgroups_per_row: n.div_ceil(tiling.block_n()),
+ threadgroups_per_column: m.div_ceil(tiling.block_m()),
+ aligned_inner_iterations: k / split_k / K_STEP,
+ ..Default::default()
+ };
+ let alignment =
+ GemmAlignment::new(m.is_multiple_of(tiling.block_m()), n.is_multiple_of(tiling.block_n()), true);
+ let key = (tiling, format, alignment);
+ let kernel = match self.kernels.entry(key) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => entry.insert(GemmTrellisMetalKernel::new(
+ command_buffer.context(),
+ tiling,
+ alignment,
+ format.vector_width,
+ format.transition_bits,
+ format.restart_columns.unwrap_or(0),
+ )?),
+ };
+ macro_rules! encode_gemm {
+ ($destination:expr) => {
+ kernel.encode(
+ activations,
+ column_group_sums,
+ activation_scales,
+ codes,
+ row_scales,
+ codebook,
+ $destination,
+ std::slice::from_ref(&params),
+ params.threadgroups_per_row,
+ params.threadgroups_per_column,
+ split_k,
+ command_buffer,
+ )
+ };
+ }
+ if split_k == 1 {
+ encode_gemm!(arguments.output.values);
+ return Ok(());
+ }
+
+ let partial_count = split_k as usize * m as usize * n as usize;
+ let mut partial_sums = command_buffer.allocate_scratch(partial_count * std::mem::size_of::<i32>())?;
+ encode_gemm!(&mut partial_sums);
+ let reduce_kernel = match &mut self.reduce_kernel {
+ Some(kernel) => kernel,
+ empty => empty.insert(GemmTrellisReduceMetalKernel::new(command_buffer.context())?),
+ };
+ reduce_kernel.encode(
+ &partial_sums,
+ column_group_sums,
+ activation_scales,
+ row_scales,
+ codebook,
+ arguments.output.values,
+ m,
+ n,
+ split_k,
+ output_stride,
+ command_buffer,
+ );
+ Ok(())
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 1e44c109a..ff949980c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -271,27 +271,27 @@ impl GemvKernel {
output_work: &MatmulOutputWork,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<(), MatmulError<Metal>> {
- let ab_scale = arguments.d_transform.ab_scale;
- let output_bias = arguments.d_transform.bias;
- let rht_factors = arguments.d_transform.rht_factors;
- let soft_cap = arguments.d_transform.soft_cap;
- let deferred_factors = rht_factors.filter(|_| !specialization.fuses_rht());
- let (gemv_bias, gemv_rht_factors) = if deferred_factors.is_some() {
- (None, None)
- } else {
- (output_bias, rht_factors)
- };
-
let MatmulArguments {
a,
b,
- mut d,
+ output,
m,
n,
k,
gather_indices,
..
} = arguments;
+ let (mut d, ops) = output.into_contiguous(n, "Gemv")?;
+ let ab_scale = ops.ab_scale;
+ let output_bias = ops.bias;
+ let rht_factors = ops.rht_factors;
+ let soft_cap = ops.soft_cap;
+ let deferred_factors = rht_factors.filter(|_| !specialization.fuses_rht());
+ let (gemv_bias, gemv_rht_factors) = if deferred_factors.is_some() {
+ (None, None)
+ } else {
+ (output_bias, rht_factors)
+ };
let MatmulA::FullPrecision {
values: a,
offset: a_offset,
@@ -364,6 +364,13 @@ impl GemvKernel {
soft_cap,
command_buffer,
),
+ MatmulB::Trellis {
+ ..
+ } => {
+ return Err(MatmulError::UnsupportedLayout {
+ path: "Gemv",
+ });
+ },
}
if let Some(factors) = deferred_factors {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
index b6e8fddda..4257c9dda 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/mod.rs
@@ -31,7 +31,7 @@ use crate::{
pub struct MatmulMetalKernel {
gemv: GemvKernel,
- pub gemm: GemmKernel,
+ gemm: GemmKernel,
output_work: MatmulOutputWork,
weights_data_type: DataType,
input_data_type: DataType,
@@ -130,6 +130,10 @@ impl MatmulMetalKernel {
let all_bf16 = weights_data_type == DataType::BF16
&& input_data_type == DataType::BF16
&& output_data_type == DataType::BF16;
+ let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);
+ if shape.b_is_trellis {
+ return MatmulDispatch::Gemm(problem.select_plan());
+ }
if let Some(route) = qmv::route(device_name, apple_gpu_family, supports_mxu, shape, all_bf16) {
return match route {
QmvRoute::Tuned(tile) | QmvRoute::MainGemv(tile) => MatmulDispatch::Gemv(
@@ -147,7 +151,6 @@ impl MatmulMetalKernel {
gpu_core_count,
apple_gpu_family,
);
- let problem = GemmProblem::new(*shape, weights_data_type, output_data_type, supports_mxu, apple_gpu_family);
let plan = problem.select_plan();
match gemv {
None => MatmulDispatch::Gemm(plan),
@@ -274,24 +277,12 @@ impl MatmulKernel for MatmulMetalKernel {
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<(), MetalError> {
let shape = MatmulShape::from_arguments(&arguments);
- let plan = match self.select_dispatch(&shape, command_buffer.context()) {
+ match self.select_dispatch(&shape, command_buffer.context()) {
MatmulDispatch::Gemv(gemv) => {
- return self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from);
+ self.gemv.encode(arguments, gemv, &self.output_work, command_buffer).map_err(MetalError::from)
},
- MatmulDispatch::Gemm(plan) => plan,
- };
-
- // TODO: remove after GatherGEMM is supported
- if arguments.gather_indices.is_some() {
- return Err(MetalError::KernelDispatchFailed(
- format!(
- "gathered readout requires the GEMV path, but shape (m={}, n={}) routes to GEMM",
- arguments.m, arguments.n
- )
- .into(),
- ));
+ MatmulDispatch::Gemm(plan) => self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer),
}
- self.gemm.encode_plan(arguments, plan, &self.output_work, command_buffer)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 3ed2ccabf..fd218c54a 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -7,7 +7,7 @@ use crate::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding, Kernels,
kernel::{
LogitTransformKernel,
- matmul::{MatmulA, MatmulArguments, MatmulDOps, MatmulKernel},
+ matmul::{MatmulA, MatmulArguments, MatmulDOps, MatmulKernel, MatmulOutput},
},
},
config::{embedding::AnyEmbeddingConfig, weight_matrix::AnyWeightMatrixSpec},
@@ -218,8 +218,7 @@ impl<B: Backend> Embedding<B> {
b: table.as_matrix().expect("tied embedding tables are matrices").matmul_b(),
b_leading_dimension: None,
b_transpose: true,
- d: &mut output,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(&mut output, MatmulDOps::none()),
gather_indices,
m: batch_dim,
n: output_dim,
diff --git a/crates/uzu-engine/src/encodable_block/linear/matmul.rs b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
index 23219f816..4f7f6a9ad 100644
--- a/crates/uzu-engine/src/encodable_block/linear/matmul.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/matmul.rs
@@ -7,7 +7,10 @@ use crate::{
Backend, BufferRef, CommandBuffer, CommandBufferEncoding,
kernel::{
ActivationQuantization, Kernels,
- matmul::{ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulShape},
+ matmul::{
+ ActivationFormat, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,
+ MatmulShape,
+ },
},
},
config::weight_matrix::{AnyWeightMatrixSpec, Layout},
@@ -131,8 +134,7 @@ impl<B: Backend> LinearMatmul<B> {
b: self.matmul_b(),
b_leading_dimension: None,
b_transpose: true,
- d: &mut output,
- d_transform: self.d_ops(),
+ output: MatmulOutput::new(&mut output, self.d_ops()),
gather_indices,
m: batch_dim,
n: output_dim,
@@ -157,6 +159,7 @@ impl<B: Backend> LinearMatmul<B> {
b_transpose: true,
b_leading_dimension: None,
b_prologue: b.b_prologue(),
+ b_is_trellis: b.is_trellis(),
b_bits: b.bits_per_b(),
b_group_size: b.group_size(),
signed_codes: b.signed_codes(),
diff --git a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
index bb818d607..d4faa24b9 100644
--- a/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/uzu-engine/src/encodable_block/linear/qlora_wrapper.rs
@@ -8,7 +8,7 @@ use crate::{
gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE,
kernel::{
ActivationTransform, Kernels,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},
},
},
config::weight_matrix::{AnyWeightMatrixSpec, hybrid_spec::IncoherenceProcessingMode, low_rank_spec::LowRankSpec},
@@ -193,8 +193,7 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
},
b_leading_dimension: None,
b_transpose: true,
- d: &mut intermediate,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(&mut intermediate, MatmulDOps::none()),
gather_indices: None::<&B::ScratchBuffer>,
m: batch_dim,
n: self.lora_rank,
@@ -235,11 +234,13 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
},
b_leading_dimension: None,
b_transpose: true,
- d: &mut output,
- d_transform: MatmulDOps {
- accumulate: true,
- ..MatmulDOps::none()
- },
+ output: MatmulOutput::new(
+ &mut output,
+ MatmulDOps {
+ accumulate: true,
+ ..MatmulDOps::none()
+ },
+ ),
gather_indices: None::<&B::ScratchBuffer>,
m: batch_dim,
n: self.output_dim,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
index 962552ffc..657f20156 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/a8w_bench.rs
@@ -15,8 +15,8 @@ use crate::{
ActivationQuantization, ActivationTransform, Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
matmul::{
- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, QuantParams,
- QuantParamsLayout, QuantizedB, QuantizedCorrection,
+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,
+ QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection,
},
},
},
@@ -148,8 +148,7 @@ impl BenchmarkData {
}),
b_leading_dimension: None,
b_transpose: true,
- d: output,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(output, MatmulDOps::none()),
gather_indices: None,
m: self.m,
n: self.n,
@@ -204,8 +203,7 @@ fn encode_step(
}),
b_leading_dimension: None,
b_transpose: true,
- d: output,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(output, MatmulDOps::none()),
gather_indices: None,
m: data.m,
n: data.n,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs
index 551730e75..a9448d97f 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemm_bench.rs
@@ -11,7 +11,7 @@ use crate::{
Backend,
kernel::{
Kernels,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},
},
},
metal::{GemmEngine, Metal},
@@ -63,8 +63,7 @@ fn bench_gemm(c: &mut Criterion) {
},
b_leading_dimension: None,
b_transpose: true,
- d: &mut d,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(&mut d, MatmulDOps::none()),
gather_indices: None::<&<Metal as Backend>::GlobalBuffer>,
m,
n,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
index 2ef96a40b..2924eacf8 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -13,7 +13,7 @@ use crate::{
gpu_types::QuantizationMethod,
kernel::{
Kernels,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},
},
},
cpu::Cpu,
@@ -88,11 +88,13 @@ fn run_gemv<B: Backend, T: ArrayElement + Float>(
b,
b_leading_dimension: None,
b_transpose: true,
- d: &mut d,
- d_transform: MatmulDOps {
- soft_cap,
- ..MatmulDOps::none()
- },
+ output: MatmulOutput::new(
+ &mut d,
+ MatmulDOps {
+ soft_cap,
+ ..MatmulDOps::none()
+ },
+ ),
gather_indices,
m: m as u32,
n: n_out as u32,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
index ab1716ec5..fe5ec76ae 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/mod.rs
@@ -6,3 +6,4 @@ mod quant_dispatch_test;
mod quant_gemm_bench;
mod quant_gemv_bench;
mod qwen3_bench;
+mod trellis_gemm_test;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index 728ba43de..494469be9 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -21,7 +21,10 @@ use crate::{
kernel::{
Kernels,
activation_transform::ACTIVATION_SCALE_GROUP_SIZE,
- matmul::{MatmulDOps, MatmulError, MatmulKernel, QuantParams, QuantParamsLayout, QuantParamsStrides},
+ matmul::{
+ MatmulDOps, MatmulError, MatmulKernel, MatmulOutput, QuantParams, QuantParamsLayout,
+ QuantParamsStrides,
+ },
},
},
cpu::Cpu,
@@ -207,7 +210,7 @@ fn parity_bf16_gs32_4bit_mlx_with_bias() {
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
let mut args = quant_arguments(&mut buffers, &input);
- args.d_transform = MatmulDOps {
+ args.output.ops = MatmulDOps {
bias: Some(&bias_pp_buf),
..MatmulDOps::none()
};
@@ -257,7 +260,7 @@ fn parity_bf16_gemv_qmv_fused_scale_bias() {
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
let mut args = quant_arguments(&mut buffers, &input);
- args.d_transform = MatmulDOps {
+ args.output.ops = MatmulDOps {
ab_scale: scale,
bias: Some(&bias_buf),
..MatmulDOps::none()
@@ -362,7 +365,7 @@ fn parity_bf16_quant_rht(
.expect("MatmulCpuKernel");
let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("cpu command buffer");
let mut cpu_args = quant_arguments(&mut cpu_buffers, &input);
- cpu_args.d_transform = MatmulDOps {
+ cpu_args.output.ops = MatmulDOps {
bias: with_bias.then_some(&cpu_bias),
rht_factors: Some(&cpu_rht),
..MatmulDOps::none()
@@ -383,7 +386,7 @@ fn parity_bf16_quant_rht(
.expect("MatmulMetalKernel");
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
let mut args = quant_arguments(&mut buffers, &input);
- args.d_transform = MatmulDOps {
+ args.output.ops = MatmulDOps {
bias: with_bias.then_some(&metal_bias),
rht_factors: Some(&metal_rht),
..MatmulDOps::none()
@@ -434,7 +437,7 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
let mut args = quant_arguments(&mut buffers, &input);
- args.d_transform = MatmulDOps {
+ args.output.ops = MatmulDOps {
accumulate: true,
..MatmulDOps::none()
};
@@ -726,7 +729,7 @@ fn a8w_mxu_output_bias_parity_bf16(
.expect("CPU matmul kernel");
let mut cpu_command_buffer = cpu_context.create_command_buffer(None, None).expect("CPU command buffer");
let mut cpu_arguments = quant_arguments(&mut cpu_buffers, &reference_input);
- cpu_arguments.d_transform = MatmulDOps {
+ cpu_arguments.output.ops = MatmulDOps {
rht_factors: cpu_output_hadamard_factors.as_ref(),
..MatmulDOps::none()
};
@@ -753,7 +756,7 @@ fn a8w_mxu_output_bias_parity_bf16(
.expect("Metal matmul kernel");
let mut metal_command_buffer = context.create_command_buffer(None, None).unwrap();
let mut metal_arguments = quant_arguments(&mut metal_buffers, &input);
- metal_arguments.d_transform = MatmulDOps {
+ metal_arguments.output.ops = MatmulDOps {
bias: Some(&metal_output_bias),
rht_factors: metal_output_hadamard_factors.as_ref(),
..MatmulDOps::none()
@@ -795,8 +798,7 @@ fn run_widened_f32<B: Backend>(
b,
b_leading_dimension: None,
b_transpose: true,
- d: &mut y,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(&mut y, MatmulDOps::none()),
gather_indices: None::<&B::GlobalBuffer>,
m: input.m,
n: input.n,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/trellis_gemm_test.rs
new file mode 100644
index 000000000..1d77d6182
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/trellis_gemm_test.rs
@@ -0,0 +1,189 @@
+#![cfg(backend = "metal")]
+
+use half::bf16;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+use rstest::rstest;
+use uzu_engine_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{
+ Backend, Context, Kernels,
+ kernel::matmul::{
+ MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput, TrellisFormat,
+ },
+ },
+ metal::{Metal, MetalContext},
+ },
+ data_type::DataType,
+ tests::{
+ helpers::{buffer_to_vec, create_buffer_with_data, submit_command_buffer},
+ util::shared_metal_context,
+ },
+};
+
+const V4_T8: TrellisFormat = TrellisFormat {
+ vector_width: 4,
+ transition_bits: 8,
+ restart_columns: Some(64),
+};
+const V2_T6: TrellisFormat = TrellisFormat {
+ vector_width: 2,
+ transition_bits: 6,
+ restart_columns: None,
+};
+const V2_T4: TrellisFormat = TrellisFormat {
+ vector_width: 2,
+ transition_bits: 4,
+ restart_columns: None,
+};
+
+fn decode_state(
+ format: TrellisFormat,
+ row: &[u8],
+ column: usize,
+) -> u16 {
+ let width = format.vector_width as usize;
+ let (block, local) =
+ format.restart_columns.map_or((0, column), |restart| (column / restart as usize, column % restart as usize));
+ let block_bytes = format.restart_columns.map_or(0, |restart| row_bytes(format, restart as usize));
+ let start_bit = block * block_bytes * 8 + local / width * format.transition_bits as usize;
+ (start_bit..start_bit + 16).fold(0u32, |state, bit| (state << 1) | u32::from(row[bit / 8] >> (7 - bit % 8) & 1))
+ as u16
+}
+
+fn row_bytes(
+ format: TrellisFormat,
+ columns: usize,
+) -> usize {
+ let block_columns = format.restart_columns.map_or(columns, |restart| restart as usize);
+ let transitions = block_columns / format.vector_width as usize - 1;
+ let bytes_per_block = (16 + transitions * format.transition_bits as usize).div_ceil(8);
+ columns.div_ceil(block_columns) * bytes_per_block
+}
+
+fn decode_level(
+ format: TrellisFormat,
+ row: &[u8],
+ column: usize,
+) -> i32 {
+ let state = u32::from(decode_state(format, row, column));
+ let hash = state.wrapping_mul(0xCFCC_B83F).wrapping_add(0x584B_4AA3);
+ let hash = (hash ^ hash >> 16).wrapping_mul(0x85EB_CA6B);
+ let hash_byte = ((hash ^ hash >> 16) >> (8 * (column % format.vector_width as usize))) & 0xFF;
+ let field_sum: u32 = (0..4).map(|field| (hash_byte >> (2 * field)) & 3).sum();
+ (8 * field_sum + ((3 * (hash_byte & 15)) & 15)) as i32 - 54
+}
+
+type MetalBuffer = <Metal as Backend>::GlobalBuffer;
+type MetalMatmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel;
+
+const CODEBOOK: [f32; 5] = [0.0123, 0.3, -0.25, 0.125, -0.0625];
+const PADDING: usize = 24;
+const SENTINEL: u16 = 0x7F7F;
+const RELATIVE_TOLERANCE: f32 = 1e-2;
+// Cases are ordered as (m, n, k).
+const CASES: [(usize, usize, usize); 7] =
+ [(1, 80, 5120), (17, 80, 5120), (512, 80, 5120), (16, 128, 64), (32, 128, 64), (1, 6, 64), (2048, 2048, 128)];
+
+fn run_projection(
+ context: &MetalContext,
+ format: TrellisFormat,
+ dimensions: (usize, usize, usize),
+) {
+ let (m, n, k) = dimensions;
+ let mut rng = SmallRng::seed_from_u64((m * k + n) as u64);
+ let stored: Vec<u8> = (0..n * row_bytes(format, k)).map(|_| rng.random()).collect();
+ let row_scales: Vec<f32> = (0..n).map(|_| rng.random_range(0.001..0.01)).collect();
+ let mut values: Vec<i8> = (0..m * k).map(|_| rng.random_range(-127..=127)).collect();
+ if m > 1 {
+ // Exercise the zero-activation path in the epilogue.
+ values[k..2 * k].fill(0);
+ }
+ let group_sums_host: Vec<f32> = values
+ .chunks_exact(k)
+ .flat_map(|row| {
+ (0..4).map(|group| row.iter().skip(group).step_by(4).fold(0.0, |sum, &value| sum + f32::from(value)))
+ })
+ .collect();
+ let activation_scales_host: Vec<f32> = (0..m).map(|_| rng.random_range(0.001..0.01)).collect();
+ let activations = create_buffer_with_data::<Metal, i8>(context, &values);
+ let group_sums = create_buffer_with_data::<Metal, f32>(context, &group_sums_host);
+ let activation_scales = create_buffer_with_data::<Metal, f32>(context, &activation_scales_host);
+ let codes = create_buffer_with_data::<Metal, u8>(context, &stored);
+ let row_scales_buffer = create_buffer_with_data::<Metal, f32>(context, &row_scales);
+ let codebook = create_buffer_with_data::<Metal, f32>(context, &CODEBOOK);
+ let mut output = create_buffer_with_data::<Metal, u16>(context, &vec![SENTINEL; m * (n + PADDING)]);
+ let arguments = MatmulArguments {
+ a: MatmulA::Trellis {
+ values: &activations,
+ column_group_sums: &group_sums,
+ scales: &activation_scales,
+ },
+ b: MatmulB::Trellis {
+ codes: &codes,
+ row_scales: &row_scales_buffer,
+ codebook: &codebook,
+ format,
+ },
+ b_leading_dimension: None,
+ b_transpose: true,
+ output: MatmulOutput {
+ values: &mut output,
+ row_stride: Some((n + PADDING) as u32),
+ ops: MatmulDOps::none(),
+ },
+ gather_indices: None::<&MetalBuffer>,
+ m: m as u32,
+ n: n as u32,
+ k: k as u32,
+ };
+ let mut matmul =
+ <MetalMatmul as MatmulKernel>::new(context, DataType::BF16, DataType::BF16, DataType::BF16).unwrap();
+ let mut command_buffer = context.create_command_buffer(None, None).unwrap();
+ matmul.encode(arguments, &mut command_buffer).unwrap();
+ submit_command_buffer(command_buffer);
+ let decoded_levels: Vec<i32> = stored
+ .chunks_exact(row_bytes(format, k))
+ .flat_map(|codes| (0..k).map(|column| decode_level(format, codes, column)))
+ .collect();
+ let actual = buffer_to_vec::<Metal, u16>(&output);
+ let stride = n + PADDING;
+ for (index, &got) in actual.iter().enumerate() {
+ let token = index / stride;
+ let row = index % stride;
+ if row < n {
+ let row_values = &values[token * k..(token + 1) * k];
+ let levels = &decoded_levels[row * k..(row + 1) * k];
+ let level_dot: i32 = levels.iter().zip(row_values).map(|(&level, &value)| level * i32::from(value)).sum();
+ let sums = &group_sums_host[token * 4..token * 4 + 4];
+ // A non-dyadic offset exposes rounding differences in the GPU dot.
+ let offsets_dot =
+ (((sums[0] * CODEBOOK[1]) + sums[1] * CODEBOOK[2]) + sums[2] * CODEBOOK[3]) + sums[3] * CODEBOOK[4];
+ let dot = (level_dot as f32).mul_add(CODEBOOK[0], offsets_dot);
+ let want = bf16::from_f32(dot * row_scales[row] * activation_scales_host[token]).to_f32();
+ let got = bf16::from_bits(got).to_f32();
+ assert!(
+ (got - want).abs() <= RELATIVE_TOLERANCE * want.abs(),
+ "{format:?} ({m}, {n}, {k}) token {token} row {row}: got {got}, want {want}"
+ );
+ } else {
+ assert_eq!(got, SENTINEL, "{format:?} M {m} N {n} K {k}, token {token}, row {row}");
+ }
+ }
+}
+
+#[rstest]
+#[test_attr(uzu_test)]
+#[case::v4_t8(V4_T8)]
+#[case::v2_t6(V2_T6)]
+#[case::v2_t4(V2_T4)]
+fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {
+ let context = shared_metal_context();
+ if !context.supports_mxu {
+ return;
+ }
+ for (m, n, k) in CASES {
+ run_projection(&context, format, (m, n, k));
+ }
+}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index 466c1a2bf..563d2d8c2 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -19,6 +19,7 @@ fn shape(
b_transpose: true,
b_leading_dimension: None,
b_prologue: GemmBPrologueKind::FullPrecision,
+ b_is_trellis: false,
b_bits: None,
b_group_size: None,
signed_codes: false,
@@ -70,33 +71,18 @@ fn policy_boundaries_are_preserved() {
assert_eq!(policy::mxu_fp_tile(m, n, k), expected);
}
- for (m, n, group_size, expected) in [
- (16, 4096, 31, Tile64x64x16_Simdgroups2x2),
- // Full M blocks keep the narrow tile.
- (4, 4096, 32, Tile8x32x32_Simdgroups1x1),
- (8, 4096, 32, Tile8x32x32_Simdgroups1x1),
- (16, 4096, 32, Tile8x32x32_Simdgroups1x1),
- (24, 4096, 32, Tile8x32x32_Simdgroups1x1),
- // A partial trailing block takes the wide tile.
- (9, 4096, 32, Tile32x32x32_Simdgroups2x2),
- (15, 4096, 32, Tile32x32x32_Simdgroups2x2),
- (31, 4096, 32, Tile32x32x32_Simdgroups2x2),
- (64, 6143, 32, Tile32x32x32_Simdgroups2x2),
- (64, 6144, 32, Tile64x64x32_Simdgroups2x2),
+ for (m, n, group_size, family, expected) in [
+ (16, 4096, 31, MTLGPUFamily::Apple7, Tile64x64x16_Simdgroups2x2),
+ (16, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
+ (8, 4096, 32, MTLGPUFamily::Apple8, Tile8x32x32_Simdgroups1x1),
+ (9, 4096, 32, MTLGPUFamily::Apple8, Tile32x32x32_Simdgroups2x2),
+ (9, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
+ (31, 4096, 32, MTLGPUFamily::Apple9, Tile8x32x32_Simdgroups1x1),
+ (32, 4096, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
+ (64, 6143, 32, MTLGPUFamily::Apple9, Tile32x32x32_Simdgroups2x2),
+ (64, 6144, 32, MTLGPUFamily::Apple9, Tile64x64x32_Simdgroups2x2),
] {
- assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, MTLGPUFamily::Apple7), expected);
- }
-
- // Apple8 retains the older wide-tile policy; Apple9 and newer keep the narrow tile.
- for apple_gpu_family in [MTLGPUFamily::Apple7, MTLGPUFamily::Apple8] {
- for m in [9, 15, 31] {
- assert_eq!(policy::simdgroup_quant_tile(m, 4096, 32, apple_gpu_family), Tile32x32x32_Simdgroups2x2);
- }
- }
- for apple_gpu_family in [MTLGPUFamily::Apple9, MTLGPUFamily::Apple10] {
- for m in [9, 15, 31] {
- assert_eq!(policy::simdgroup_quant_tile(m, 4096, 32, apple_gpu_family), Tile8x32x32_Simdgroups1x1);
- }
+ assert_eq!(policy::simdgroup_quant_tile(m, n, group_size, family), expected);
}
}
@@ -163,6 +149,24 @@ fn selection_fallbacks_and_split_k_are_preserved() {
assert_eq!(problem(zero, DataType::BF16).select_plan().split_k, 1);
}
+#[uzu_test]
+fn trellis_plan_matches_projection_cases() {
+ use GemmTiling::*;
+
+ for (m, n, k, tiling, split_k) in [
+ (16, 128, 64, Tile16x32x256_Simdgroups1x1, 1),
+ (17, 80, 64, Tile64x64x256_Simdgroups2x2, 1),
+ (2048, 2048, 128, Tile128x128x256_Simdgroups4x4, 1),
+ (1, 80, 5120, Tile16x32x256_Simdgroups1x1, 80),
+ ] {
+ let mut trellis_shape = shape(m, n, k);
+ trellis_shape.a_full_precision = false;
+ trellis_shape.b_is_trellis = true;
+ let plan = problem(trellis_shape, DataType::BF16).select_plan();
+ assert_eq!((plan.engine, plan.tiling, plan.split_k), (GemmEngine::Mxu, tiling, split_k), "M {m} N {n} K {k}");
+ }
+}
+
#[uzu_test]
fn forced_engine_errors_are_preserved() {
let huge = shape(u32::MAX, u32::MAX, u32::MAX);
@@ -176,7 +180,7 @@ fn forced_engine_errors_are_preserved() {
invalid_layout.b_transpose = false;
assert_eq!(
problem(invalid_layout, DataType::BF16).select_plan_for_engine(GemmEngine::Mxu),
- Err(GemmPlanError::UnsupportedQuantLayout)
+ Err(GemmPlanError::UnsupportedLayout("quantized weights require transposed contiguous B"))
);
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
index 2c158982a..8e2264b52 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemv/policy_test.rs
@@ -119,6 +119,7 @@ fn quant_shape(
b_transpose: true,
b_leading_dimension: None,
b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,
+ b_is_trellis: false,
b_bits: Some(4),
b_group_size: Some(32),
signed_codes: false,
@@ -138,6 +139,7 @@ fn block_unaligned_quantized_k_stays_on_gemv() {
b_transpose: true,
b_leading_dimension: None,
b_prologue: GemmBPrologueKind::ScaleZeroPointDequant,
+ b_is_trellis: false,
b_bits: Some(4),
b_group_size: Some(64),
signed_codes: false,
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
index 5684be512..9121e6a80 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/qmv/routes_test.rs
@@ -1,5 +1,3 @@
-use std::fmt::Write;
-
use metal::MTLGPUFamily;
use uzu_engine_macros::uzu_test;
use xxhash_rust::xxh3::xxh3_64;
@@ -58,6 +56,7 @@ fn problem(
b_transpose: true,
b_leading_dimension: None,
b_prologue: prologue,
+ b_is_trellis: false,
b_bits: Some(bits),
b_group_size: Some(group),
signed_codes: false,
@@ -72,9 +71,6 @@ fn problem(
fn table_is_complete_and_fingerprint_is_stable() {
let mut canonical = Vec::new();
let mut matched_rows = vec![false; ROWS.len()];
- let mut tuned = 0;
- let mut main_gemv = 0;
- let mut main_gemm = 0;
for &(device_label, device_name, gpu_core_count, apple_gpu_family, supports_mxu) in &DEVICES {
for &(format_name, bits, group, prologue) in &FORMATS {
for m in 2..=7 {
@@ -96,11 +92,6 @@ fn table_is_complete_and_fingerprint_is_stable() {
let (row_index, row) = matches[0];
matched_rows[row_index] = true;
let selected = row.route;
- match selected {
- QmvRoute::Tuned(_) => tuned += 1,
- QmvRoute::MainGemv(_) => main_gemv += 1,
- QmvRoute::MainGemm(_) => main_gemm += 1,
- }
canonical.push(format!("{device_label}|{format_name}|{shape_name}|{m}|{n}|{k}|{selected:?}"));
let problem = problem(m, n, k, bits, group, prologue);
assert_eq!(route(device_name, apple_gpu_family, supports_mxu, &problem, true), Some(selected));
@@ -134,14 +125,7 @@ fn table_is_complete_and_fingerprint_is_stable() {
assert!(matched_rows.into_iter().all(|matched| matched), "route table contains an orphaned row");
canonical.sort();
assert_eq!(canonical.len(), 1176);
- let mut expanded = String::new();
- for line in canonical {
- writeln!(&mut expanded, "{line}").expect("writing to a String must succeed");
- }
- assert_eq!(tuned, 864);
- assert_eq!(main_gemv, 221);
- assert_eq!(main_gemm, 91);
- assert_eq!(xxh3_64(expanded.trim_end().as_bytes()), FROZEN_PLAN_FINGERPRINT);
+ assert_eq!(xxh3_64(canonical.join("\n").as_bytes()), FROZEN_PLAN_FINGERPRINT);
}
#[uzu_test]
@@ -229,23 +213,18 @@ fn normal_routing_handles_inputs_outside_the_frozen_matrix() {
#[uzu_test]
fn family_lookup_requires_one_unanimous_route() {
let m1_route = problem(4, 8192, 5120, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- let measured_m1 = route("Apple M1", MTLGPUFamily::Apple7, false, &m1_route, true);
- for device_name in ["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"] {
- assert_eq!(route(device_name, MTLGPUFamily::Apple7, false, &m1_route, true), measured_m1);
- }
-
let unanimous = problem(6, 5120, 17408, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
- let measured_m2 = route("Apple M2", MTLGPUFamily::Apple8, false, &unanimous, true);
- for device_name in ["Apple M2 Max", "Apple M2 Ultra"] {
- assert_eq!(route(device_name, MTLGPUFamily::Apple8, false, &unanimous, true), measured_m2);
- }
- let measured_m3_max = route("Apple M3 Max", MTLGPUFamily::Apple9, false, &unanimous, true);
- for device_name in ["Apple M3", "Apple M3 Pro", "Apple M4 Max"] {
- assert_eq!(route(device_name, MTLGPUFamily::Apple9, false, &unanimous, true), measured_m3_max);
- }
- let measured_m5_max = route("Apple M5 Max", MTLGPUFamily::Apple10, true, &unanimous, true);
- for device_name in ["Apple M5", "Apple M5 Pro"] {
- assert_eq!(route(device_name, MTLGPUFamily::Apple10, true, &unanimous, true), measured_m5_max);
+ let families: [(&MatmulShape, MTLGPUFamily, bool, &str, &[&str]); 4] = [
+ (&m1_route, MTLGPUFamily::Apple7, false, "Apple M1", &["Apple M1 Pro", "Apple M1 Max", "Apple M1 Ultra"]),
+ (&unanimous, MTLGPUFamily::Apple8, false, "Apple M2", &["Apple M2 Max", "Apple M2 Ultra"]),
+ (&unanimous, MTLGPUFamily::Apple9, false, "Apple M3 Max", &["Apple M3", "Apple M3 Pro", "Apple M4 Max"]),
+ (&unanimous, MTLGPUFamily::Apple10, true, "Apple M5 Max", &["Apple M5", "Apple M5 Pro"]),
+ ];
+ for (shape, family, supports_mxu, measured_device, aliases) in families {
+ let measured = route(measured_device, family, supports_mxu, shape, true);
+ for device in aliases {
+ assert_eq!(route(device, family, supports_mxu, shape, true), measured);
+ }
}
let disagreement = problem(3, 5120, 6144, 4, 64, GemmBPrologueKind::ScaleZeroPointDequant);
diff --git a/crates/uzu-engine/unit/common/matmul/harness.rs b/crates/uzu-engine/unit/common/matmul/harness.rs
index 84878e69f..c729c08f3 100644
--- a/crates/uzu-engine/unit/common/matmul/harness.rs
+++ b/crates/uzu-engine/unit/common/matmul/harness.rs
@@ -10,7 +10,7 @@ use crate::{
Backend, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context,
kernel::{
Kernels,
- matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
+ matmul::{MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput},
},
},
cpu::Cpu,
@@ -170,8 +170,7 @@ fn run<B: Backend, T: ArrayElement + Float>(
},
b_leading_dimension: None,
b_transpose: input.case.b_transpose,
- d: &mut d_buffer,
- d_transform,
+ output: MatmulOutput::new(&mut d_buffer, d_transform),
gather_indices: None,
m,
n,
diff --git a/crates/uzu-engine/unit/common/matmul/quant.rs b/crates/uzu-engine/unit/common/matmul/quant.rs
index 482677f87..1149155d0 100644
--- a/crates/uzu-engine/unit/common/matmul/quant.rs
+++ b/crates/uzu-engine/unit/common/matmul/quant.rs
@@ -17,8 +17,8 @@ use crate::{
kernel::{
ActivationQuantization, ActivationTransform, Kernels,
matmul::{
- Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, QuantParams,
- QuantParamsLayout, QuantizedB, QuantizedCorrection,
+ Int8CodeLayout, MatmulA, MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulOutput,
+ QuantParams, QuantParamsLayout, QuantizedB, QuantizedCorrection,
},
},
},
@@ -445,8 +445,7 @@ pub fn quant_arguments<'a, B: Backend, T: ArrayElement + Float>(
b,
b_leading_dimension: None,
b_transpose: true,
- d: y,
- d_transform: MatmulDOps::none(),
+ output: MatmulOutput::new(y, MatmulDOps::none()),
gather_indices: None,
m: input.m,
n: input.n,