Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support Trellis decode for pre M5s (#1071)

CCYeh23ac62825968
diff --git a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
index 55bca59a2..637e75054 100644
--- a/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
+++ b/crates/uzu-engine/src/backends/metal/gpu_types/gemm.rs
@@ -38,6 +38,7 @@ pub enum GemmTiling {
}
pub const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
+pub const TRELLIS_DECODE_MAX_BATCH: u32 = 8;
impl GemmTiling {
pub const fn is_packed_w4(self) -> bool {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
index 16b699a07..59b475209 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/gemm.h
@@ -56,6 +56,8 @@ enum class GemmTiling : uint32_t {
static constant constexpr uint32_t MXU_SIMDGROUP_BLOCK_K = 32;
+static constant constexpr uint32_t TRELLIS_DECODE_MAX_BATCH = 8;
+
struct GemmAlignment {
uint32_t raw_value;
constexpr GemmAlignment() thread : raw_value(0) {}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
index 9480e6012..15e4d921c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/quant_unpack.h
@@ -129,5 +129,15 @@ inline void dequantize(
}
}
+// A byte ored with 0x6400 is the half 1024 plus the byte, so the subtraction is exact.
+METAL_FUNC float4 bytes_to_float(const uint word, const float offset) {
+ const half2 bias = half2(half(1024.0f + offset));
+ const half2 even = as_type<half2>((word & 0x00FF00FFu) | 0x64006400u) - bias;
+ const half2 odd = as_type<half2>(((word >> 8) & 0x00FF00FFu) | 0x64006400u) - bias;
+ return float4(float(even.x), float(odd.x), float(even.y), float(odd.y));
+}
+
+METAL_FUNC float4 int8_to_float(const uint word) { return bytes_to_float(word ^ 0x80808080u, 128.0f); }
+
} // namespace gemm
} // namespace uzu
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
index 6086c2013..2383d9fd6 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis.metal
@@ -100,16 +100,14 @@ KERNEL(GemmTrellis)(
device bfloat* output,
const constant uzu::matmul::GemmParams* params,
threadgroup half stage[TRELLIS_STAGE_ELEMENTS],
- const constant uint& group_count_x,
- const constant uint& group_count_y,
- const constant uint& group_count_z,
+ const constant uint& partition_count,
const GemmAlignment alignment SPECIALIZE,
- const uint trellis_vector_width SPECIALIZE,
- const uint trellis_transition_bits SPECIALIZE,
- const uint trellis_restart_columns SPECIALIZE,
- const uint group_x GROUPS(group_count_x),
- const uint group_y GROUPS(group_count_y),
- const uint group_z GROUPS(group_count_z),
+ const uint vector_width SPECIALIZE,
+ const uint transition_bits SPECIALIZE,
+ const uint restart_columns SPECIALIZE,
+ const uint group_x GROUPS(params[0].threadgroups_per_row),
+ const uint group_y GROUPS(params[0].threadgroups_per_column),
+ const uint group_z GROUPS(partition_count),
const uint thread_x THREADS(METAL_SIMD_SIZE),
const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),
const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),
@@ -128,11 +126,7 @@ KERNEL(GemmTrellis)(
constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / Ops::FRAGMENT_ROWS;
constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / Ops::FRAGMENT_COLS;
using Products = uzu::matmul::Fragment<int, TILES_M, TILES_N, Ops>;
- const trellis::GemmTrellisFormat trellis_format{
- trellis_vector_width,
- trellis_transition_bits,
- trellis_restart_columns
- };
+ const trellis::GemmTrellisFormat trellis_format{vector_width, transition_bits, restart_columns};
const uint simdgroup = thread_context.simdgroup_index;
const uint token_base = group_y * BLOCK_M + SIMDGROUP_BLOCK_M * (simdgroup / SIMDGROUPS_PER_COLUMN);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis_decode.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis_decode.metal
new file mode 100644
index 000000000..aa779d936
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm_trellis_decode.metal
@@ -0,0 +1,105 @@
+#include "../../common/dsl.h"
+#include "../common/quant_unpack.h"
+#include "common/quantized/trellis.h"
+
+using namespace metal;
+using namespace uzu::gemm;
+
+#define SIMDGROUPS 8
+#define ROWS_PER_SIMDGROUP 4
+
+UZU_CONST uint ROWS_PER_THREADGROUP = ROWS_PER_SIMDGROUP * SIMDGROUPS;
+UZU_CONST uint MAX_BATCH = TRELLIS_DECODE_MAX_BATCH;
+
+// Pre-M5 batch 1..8. Lane partials are exact integers in float, so the cross-lane sum is an int simd_sum.
+KERNEL(GemmTrellisDecode)(
+ const device uint8_t* activations,
+ const device float4* column_group_sums,
+ const device float* activation_scales,
+ const device uint8_t* codes,
+ const device float* row_scales,
+ const device float* scale_and_offsets,
+ device bfloat* output,
+ const constant uint& k,
+ const constant uint& n,
+ const constant uint& output_stride,
+ const uint batch SPECIALIZE,
+ const uint vector_width SPECIALIZE,
+ const uint transition_bits SPECIALIZE,
+ const uint restart_columns SPECIALIZE,
+ const uint row_block GROUPS(n.div_ceil(ROWS_PER_SIMDGROUP* SIMDGROUPS)),
+ const uint lane THREADS(METAL_SIMD_SIZE),
+ const uint simdgroup THREADS(SIMDGROUPS)
+) {
+ const trellis::GemmTrellisFormat format{vector_width, transition_bits, restart_columns};
+ const uint row_bytes = trellis::row_bytes(format, k);
+ const uint first_row = row_block * ROWS_PER_THREADGROUP + simdgroup * ROWS_PER_SIMDGROUP;
+ // Rows past n re-read the last row.
+ uint row_offsets[ROWS_PER_SIMDGROUP];
+ if (batch <= 2) {
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {
+ row_offsets[row] = min(first_row + row, n - 1) * row_bytes;
+ }
+ }
+ float sums[MAX_BATCH][ROWS_PER_SIMDGROUP] = {};
+
+ for (uint column = lane * trellis::LANE_COLUMNS; column < k; column += METAL_SIMD_SIZE * trellis::LANE_COLUMNS) {
+ uint4 words[ROWS_PER_SIMDGROUP];
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {
+ words[row] = trellis::decode_biased_weight_quad(
+ format,
+ codes,
+ batch <= 2 ? row_offsets[row] : min(first_row + row, n - 1) * row_bytes,
+ column,
+ k
+ );
+ }
+ uint4 inputs[MAX_BATCH];
+ METAL_PRAGMA_UNROLL
+ for (uint token = 0; token < MAX_BATCH; ++token) {
+ if (token < batch) {
+ inputs[token] = *reinterpret_cast<const device uint4*>(activations + size_t(token) * k + column);
+ }
+ }
+ METAL_PRAGMA_UNROLL
+ for (uint quad = 0; quad < 4; ++quad) {
+ float4 weights[ROWS_PER_SIMDGROUP];
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {
+ weights[row] = bytes_to_float(words[row][quad], float(trellis::WEIGHT_BIAS));
+ }
+ METAL_PRAGMA_UNROLL
+ for (uint token = 0; token < MAX_BATCH; ++token) {
+ if (token < batch) {
+ const float4 input = int8_to_float(inputs[token][quad]);
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {
+ METAL_PRAGMA_UNROLL
+ for (uint lane_column = 0; lane_column < 4; ++lane_column) {
+ sums[token][row] = fma(weights[row][lane_column], input[lane_column], sums[token][row]);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ const float4 offsets = float4(scale_and_offsets[1], scale_and_offsets[2], scale_and_offsets[3], scale_and_offsets[4]);
+ METAL_PRAGMA_UNROLL
+ for (uint token = 0; token < MAX_BATCH; ++token) {
+ if (token < batch) {
+ const float offsets_dot = dot(column_group_sums[token], offsets);
+ METAL_PRAGMA_UNROLL
+ for (uint row = 0; row < ROWS_PER_SIMDGROUP; ++row) {
+ const float level_dot = float(simd_sum(int(sums[token][row])));
+ if (lane == 0 && first_row + row < n) {
+ output[token * output_stride + first_row + row] = bfloat(
+ (level_dot * scale_and_offsets[0] + offsets_dot) * row_scales[first_row + row] * activation_scales[token]
+ );
+ }
+ }
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
index d536dee8f..3b7c6ed60 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/trellis.rs
@@ -9,11 +9,11 @@ use crate::backends::{
Metal,
command_buffer::MetalCommandBufferEncoding,
gpu_types::{
- gemm::{GemmAlignment, GemmTiling},
+ gemm::{GemmAlignment, GemmTiling, TRELLIS_DECODE_MAX_BATCH},
matmul::GemmParams,
},
kernel::{
- GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel,
+ GemmTrellisDecodeMetalKernel, GemmTrellisMetalKernel, GemmTrellisReduceMetalKernel,
matmul::{
MatmulMetalError,
gemm::{GemmEngine, GemmPlan, selection::TRELLIS_K_STEP as K_STEP},
@@ -22,9 +22,16 @@ use crate::backends::{
},
};
+const LANE_COLUMNS: u32 = 16;
+const MAX_CENTERED_WEIGHT: u32 = 57;
+const MAX_ACTIVATION: u32 = 128;
+const DECODE_MAX_EXACT_K: u32 =
+ ((1 << 24) - 1) / (LANE_COLUMNS * MAX_CENTERED_WEIGHT * MAX_ACTIVATION) * 32 * LANE_COLUMNS;
+
#[derive(Default)]
pub(super) struct TrellisGemm {
kernels: HashMap<(GemmTiling, TrellisFormat, GemmAlignment), GemmTrellisMetalKernel>,
+ decode_kernels: HashMap<(u32, TrellisFormat), GemmTrellisDecodeMetalKernel>,
reduce_kernel: Option<GemmTrellisReduceMetalKernel>,
}
@@ -74,6 +81,36 @@ impl TrellisGemm {
return Err(unsupported());
}
+ if plan.engine == GemmEngine::Simdgroup
+ && (1..=TRELLIS_DECODE_MAX_BATCH).contains(&m)
+ && k <= DECODE_MAX_EXACT_K
+ {
+ let kernel = match self.decode_kernels.entry((m, format)) {
+ Entry::Occupied(entry) => entry.into_mut(),
+ Entry::Vacant(entry) => entry.insert(GemmTrellisDecodeMetalKernel::new(
+ command_buffer.context(),
+ m,
+ format.vector_width,
+ format.transition_bits,
+ format.restart_columns.unwrap_or(0),
+ )?),
+ };
+ kernel.encode(
+ activations,
+ column_group_sums,
+ activation_scales,
+ codes,
+ row_scales,
+ codebook,
+ arguments.output.values,
+ k,
+ n,
+ output_stride,
+ command_buffer,
+ );
+ return Ok(());
+ }
+
let params = GemmParams {
M: m,
N: n,
@@ -111,8 +148,6 @@ impl TrellisGemm {
codebook,
$destination,
std::slice::from_ref(&params),
- params.threadgroups_per_row,
- params.threadgroups_per_column,
split_k,
command_buffer,
)
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
index d0926bbeb..a47713008 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/trellis_gemm_test.rs
@@ -243,6 +243,10 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {
for dimensions in CASES.into_iter().filter(|&(_, _, k)| whole_blocks(k)) {
run_projection(&context, format, dimensions, None);
}
+ // Simdgroup plans with m up to 8 run the decode kernel. K fits every restart size.
+ for m in 1..=8 {
+ run_projection(&context, format, (m, 37, 4160), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 1)));
+ }
// The forced plans' K are whole 64-column blocks.
if whole_blocks(64) {
if context.supports_mxu {
@@ -252,7 +256,7 @@ fn trellis_projection_matches_cpu_reference(#[case] format: TrellisFormat) {
run_projection(&context, format, (17, 80, 192), Some((GemmTiling::Tile16x32x256_Simdgroups1x1, 1)));
run_projection(&context, format, (7, 80, 192), Some((GemmTiling::Tile16x32x256_Simdgroups1x1, 1)));
}
- run_projection(&context, format, (7, 68, 6144), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 4)));
+ run_projection(&context, format, (9, 68, 6144), Some((GemmTiling::Tile16x32x32_Simdgroups2x1, 4)));
run_projection(&context, format, (33, 68, 6144), Some((GemmTiling::Tile32x32x32_Simdgroups2x2, 4)));
run_projection(&context, format, (65, 68, 6144), Some((GemmTiling::Tile64x64x32_Simdgroups2x2, 4)));
}