A high-performance inference engine for AI models
Trellis activation transform (#884)
CCYeh53ef316c0359
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rsindex a8184e988..628bd0dbe 100644--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs@@ -16,6 +16,7 @@ pub mod matmul;pub mod quantization;pub mod quantization_method;pub mod ring;+pub mod trellis;pub mod trie;pub mod weaver;diff --git a/crates/uzu-engine/src/backends/common/gpu_types/trellis.rs b/crates/uzu-engine/src/backends/common/gpu_types/trellis.rsnew file mode 100644index 000000000..f2d7282fb--- /dev/null+++ b/crates/uzu-engine/src/backends/common/gpu_types/trellis.rs@@ -0,0 +1 @@+pub const COLUMN_GROUP_COUNT: u32 = 4;diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rsindex 3b665c52a..bc85140b3 100644--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs@@ -9,10 +9,12 @@ pub mod delta_net_tree_verify;pub mod gated_act_mul;pub mod matmul;pub mod radix_top_k_small;+mod trellis;pub use activation_transform::{ActivationQuantization, ActivationTransform};pub use attention::{AttentionArguments, AttentionKernel, AttentionKernelConfig};pub use gated_act_mul::{GatedActMul, GatedActMulSettings};+pub use trellis::mixing_dimension;include!(concat!(env!("OUT_DIR"), "/traits.rs"));diff --git a/crates/uzu-engine/src/backends/common/kernel/trellis.rs b/crates/uzu-engine/src/backends/common/kernel/trellis.rsnew file mode 100644index 000000000..1b3f1e4b6--- /dev/null+++ b/crates/uzu-engine/src/backends/common/kernel/trellis.rs@@ -0,0 +1,4 @@+/// Mixing matrix side length: columns divided by their largest power-of-two factor+pub fn mixing_dimension(columns: u32) -> u32 {+ columns >> columns.trailing_zeros()+}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rsindex c6e0ce5ae..28b1bfe8c 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rs@@ -1,6 +1,5 @@pub mod activation_transform;--use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;+pub(super) mod trellis;pub const INT8_SYMMETRIC_QUANTIZATION_MAXIMUM: f32 = 127.0;@@ -24,10 +23,10 @@ pub fn quantize_symmetric_i8((value / divisor).round().clamp(-INT8_SYMMETRIC_QUANTIZATION_MAXIMUM, INT8_SYMMETRIC_QUANTIZATION_MAXIMUM) as i8}-pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize]) {+pub fn hadamard_transform(values: &mut [f32]) {let mut stride = 1;- while stride < HADAMARD_TRANSFORM_BLOCK_SIZE as usize {- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {+ while stride < values.len() {+ for lane in 0..values.len() {if lane & stride == 0 {let a = values[lane];let b = values[lane | stride];@@ -37,7 +36,7 @@ pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZ}stride <<= 1;}- let scale = 1.0 / (HADAMARD_TRANSFORM_BLOCK_SIZE as f32).sqrt();+ let scale = 1.0 / (values.len() as f32).sqrt();for v in values.iter_mut() {*v *= scale;}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rsnew file mode 100644index 000000000..3455d9099--- /dev/null+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs@@ -0,0 +1,71 @@+use half::bf16;+use uzu_engine_macros::kernel;++use super::{hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8};+use crate::backends::common::{gpu_types::trellis::COLUMN_GROUP_COUNT, kernel::mixing_dimension};++fn rotate_token(+ input: &[bf16],+ rht_factors: &[f32],+ mixing: &[f32],+ mixing_dimension: usize,+) -> Vec<f32> {+ let columns = input.len();+ let hadamard_size = columns / mixing_dimension;+ let mut rotated = vec![0.0f32; columns];+ for output_mixing_index in 0..mixing_dimension {+ let mut hadamard_values: Vec<f32> = (0..hadamard_size)+ .map(|hadamard_index| {+ (0..mixing_dimension).fold(0.0f32, |accumulated, mixing_index| {+ let column = hadamard_index * mixing_dimension + mixing_index;+ (input[column].to_f32() * rht_factors[column])+ .mul_add(mixing[output_mixing_index * mixing_dimension + mixing_index], accumulated)+ })+ })+ .collect();+ hadamard_transform(&mut hadamard_values);+ for (hadamard_index, value) in hadamard_values.into_iter().enumerate() {+ rotated[hadamard_index * mixing_dimension + output_mixing_index] = bf16::from_f32(value).to_f32();+ }+ }+ rotated+}++#[kernel(TrellisTransform)]+#[variants(DIMENSION, 5120, 6144, 17408)]+pub fn trellis_transform<const DIMENSION: u32>(+ input: *const bf16,+ rht_factors: *const f32,+ mixing: *const f32,+ activations: *mut i8,+ column_group_sums: *mut f32,+ scales: *mut f32,+ batch: u32,+) {+ let columns = DIMENSION as usize;+ let mixing_dimension = mixing_dimension(DIMENSION) as usize;+ let rht_factors = unsafe { std::slice::from_raw_parts(rht_factors, columns) };+ let mixing = unsafe { std::slice::from_raw_parts(mixing, mixing_dimension * mixing_dimension) };++ for token in 0..batch as usize {+ let row = unsafe { std::slice::from_raw_parts(input.add(token * columns), columns) };+ let quantized_row = unsafe { std::slice::from_raw_parts_mut(activations.add(token * columns), columns) };+ let column_group_sums_row = unsafe {+ std::slice::from_raw_parts_mut(+ column_group_sums.add(COLUMN_GROUP_COUNT as usize * token),+ COLUMN_GROUP_COUNT as usize,+ )+ };++ let rotated = rotate_token(row, rht_factors, mixing, mixing_dimension);+ let scale = min_max_symmetric_divisor(&rotated);+ let mut token_column_group_sums = [0i32; COLUMN_GROUP_COUNT as usize];+ for (column, (quantized, &value)) in quantized_row.iter_mut().zip(&rotated).enumerate() {+ *quantized = quantize_symmetric_i8(value, scale);+ token_column_group_sums[column % COLUMN_GROUP_COUNT as usize] += i32::from(*quantized);+ }++ column_group_sums_row.copy_from_slice(&token_column_group_sums.map(|sum| sum as f32));+ unsafe { *scales.add(token) = scale };+ }+}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rsindex c4956c05f..16399ee54 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs@@ -126,7 +126,7 @@ pub fn normalization<// Input RHT like the Metal kernel: apply the factors, then transform each block, before output scalingif let Some(factors) = hadamard_factors {for block_start in (0..element_count).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {- let mut block = std::array::from_fn(|lane| {+ let mut block: [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize] = std::array::from_fn(|lane| {let factor = unsafe { *factors.add(block_start + lane) } as f32;row[block_start + lane].to_f32().unwrap() * factor});diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metalindex a23827380..b4df7bf5d 100644--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal@@ -81,9 +81,7 @@ PUBLIC KERNEL(ActivationTransform)([](float x, float y) { return max(x, y); });const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;-- const int8_t code =- static_cast<int8_t>(clamp(round(value / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));+ const int8_t code = quantize_activation_int8(value, scale);if (in_bounds) {const uint output_factor_index = grouped_by_weight_nibble ? nibble_grouped_index(factor_index) : factor_index;q_out[batch_index * element_count + output_factor_index] = code;diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metalnew file mode 100644index 000000000..0ea879905--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal@@ -0,0 +1,259 @@+#include <metal_stdlib>+#include "../common/activation_quantization.h"+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../generated/trellis.h"++using namespace metal;+namespace trellis = uzu::trellis;++#define THREADS_PER_THREADGROUP 512++UZU_CONST uint SIMDGROUPS_PER_THREADGROUP = THREADS_PER_THREADGROUP / METAL_SIMD_SIZE;++UZU_CONST uint MAX_MIXING_DIMENSION = 17;++static METAL_FUNC float butterfly(float value, ushort lane, ushort lane_stride) {+ const float other = simd_shuffle_xor(value, lane_stride);+ return (lane & lane_stride) != 0 ? other - value : value + other;+}++static METAL_FUNC uint bank_conflict_free_offset(uint hadamard_index) {+ return hadamard_index + hadamard_index / METAL_SIMD_SIZE;+}++static METAL_FUNC uint hadamard_index_before_transpose(ushort lane, ushort simdgroup, uint value_index) {+ return METAL_SIMD_SIZE * (simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index) + lane;+}++template <uint HADAMARD_SIZE>+static METAL_FUNC uint hadamard_index_after_transpose(ushort lane, ushort simdgroup, uint value_index) {+ if (HADAMARD_SIZE == 1024) {+ return METAL_SIMD_SIZE * lane + simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index;+ }+ return METAL_SIMD_SIZE * (lane + METAL_SIMD_SIZE * (value_index % 2)) + simdgroup ++ SIMDGROUPS_PER_THREADGROUP * (value_index / 2);+}++template <uint DIMENSION>+struct TransformLayout {+ static constant constexpr uint HADAMARD_SIZE = DIMENSION & (0u - DIMENSION);+ static constant constexpr uint MIXING_DIMENSION = DIMENSION / HADAMARD_SIZE;+ static constant constexpr uint VALUES_PER_THREAD = HADAMARD_SIZE / THREADS_PER_THREADGROUP;+ static constant constexpr uint COLUMNS_PER_PASS = HADAMARD_SIZE == 2048 ? 1 : (MIXING_DIMENSION > 8 ? 4 : 2);+ static constant constexpr uint COLUMN_SCRATCH_SIZE = HADAMARD_SIZE + HADAMARD_SIZE / METAL_SIMD_SIZE;+ static constant constexpr uint SCRATCH_SIZE = COLUMNS_PER_PASS * COLUMN_SCRATCH_SIZE;+ typedef float SignFlippedInput[VALUES_PER_THREAD][MIXING_DIMENSION];+ typedef float RotatedColumns[MIXING_DIMENSION][VALUES_PER_THREAD];+};++template <typename Layout, uint COLUMNS>+static METAL_FUNC void hadamard_transform_lanes(+ thread float (&column_values)[COLUMNS][Layout::VALUES_PER_THREAD],+ const ushort lane+) {+ METAL_PRAGMA_UNROLL+ for (ushort lane_stride = 1; lane_stride <= METAL_SIMD_SIZE / 2; lane_stride <<= 1) {+ METAL_PRAGMA_UNROLL+ for (uint column = 0; column < COLUMNS; ++column) {+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {+ column_values[column][value_index] = butterfly(column_values[column][value_index], lane, lane_stride);+ }+ }+ }+}++template <typename Layout, uint COLUMNS>+static METAL_FUNC void rotate_columns(+ const uint first_output_mixing_index,+ const thread typename Layout::SignFlippedInput& sign_flipped_input,+ const threadgroup float* shared_mixing,+ threadgroup float* scratch,+ thread typename Layout::RotatedColumns& rotated,+ thread float& local_maximum,+ const ushort lane,+ const ushort simdgroup+) {+ const float normalization = 1.0f / sqrt(float(Layout::HADAMARD_SIZE));++ float column_values[COLUMNS][Layout::VALUES_PER_THREAD] = {};+ METAL_PRAGMA_UNROLL+ for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {+ METAL_PRAGMA_UNROLL+ for (uint column = 0; column < COLUMNS; ++column) {+ const float mixing =+ shared_mixing[(first_output_mixing_index + column) * Layout::MIXING_DIMENSION + mixing_index];+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {+ column_values[column][value_index] =+ fma(sign_flipped_input[value_index][mixing_index], mixing, column_values[column][value_index]);+ }+ }+ }+ hadamard_transform_lanes<Layout, COLUMNS>(column_values, lane);+ METAL_PRAGMA_UNROLL+ for (uint column = 0; column < COLUMNS; ++column) {+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {+ scratch+ [column * Layout::COLUMN_SCRATCH_SIZE ++ bank_conflict_free_offset(hadamard_index_before_transpose(lane, simdgroup, value_index))] =+ column_values[column][value_index];+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ METAL_PRAGMA_UNROLL+ for (uint column = 0; column < COLUMNS; ++column) {+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {+ const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);+ column_values[column][value_index] =+ scratch[column * Layout::COLUMN_SCRATCH_SIZE + bank_conflict_free_offset(hadamard_index)];+ }+ }+ hadamard_transform_lanes<Layout, COLUMNS>(column_values, lane);+ if constexpr (Layout::HADAMARD_SIZE == 2048) {+ METAL_PRAGMA_UNROLL+ for (uint column = 0; column < COLUMNS; ++column) {+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; value_index += 2) {+ const float lower = column_values[column][value_index];+ const float upper = column_values[column][value_index + 1];+ column_values[column][value_index] = lower + upper;+ column_values[column][value_index + 1] = lower - upper;+ }+ }+ }+ METAL_PRAGMA_UNROLL+ for (uint column = 0; column < COLUMNS; ++column) {+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {+ rotated[first_output_mixing_index + column][value_index] =+ float(bfloat(column_values[column][value_index] * normalization));+ local_maximum = max(local_maximum, abs(rotated[first_output_mixing_index + column][value_index]));+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+}++// TODO: a columns agnostic kernel+template <uint DIMENSION>+VARIANTS(DIMENSION, 5120, 6144, 17408)+PUBLIC KERNEL(TrellisTransform)(+ device const bfloat* input,+ device const float* rht_factors,+ device const float* mixing,+ device int8_t* activations,+ device float4* column_group_sums,+ device float* scales,+ constant uint& batch,+ threadgroup float scratch[TransformLayout<DIMENSION>::SCRATCH_SIZE],+ threadgroup float simdgroup_maxima[SIMDGROUPS_PER_THREADGROUP],+ threadgroup float shared_mixing[MAX_MIXING_DIMENSION * MAX_MIXING_DIMENSION],+ const uint token GROUPS(batch),+ const uint thread_index THREADS(THREADS_PER_THREADGROUP),+ const ThreadContext thread_context+) {+ using Layout = TransformLayout<DIMENSION>;+ static_assert(Layout::HADAMARD_SIZE == 1024 || Layout::HADAMARD_SIZE == 2048, "unsupported Hadamard size");+ static_assert(Layout::MIXING_DIMENSION <= MAX_MIXING_DIMENSION, "mixing matrix does not fit threadgroup memory");++ const ushort lane = ushort(thread_context.simd_lane_id);+ const ushort simdgroup = ushort(thread_context.simdgroup_index);++ for (uint index = thread_index; index < Layout::MIXING_DIMENSION * Layout::MIXING_DIMENSION;+ index += THREADS_PER_THREADGROUP) {+ shared_mixing[index] = mixing[index];+ }++ typename Layout::SignFlippedInput sign_flipped_input;+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {+ const uint hadamard_index = hadamard_index_before_transpose(lane, simdgroup, value_index);+ METAL_PRAGMA_UNROLL+ for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {+ sign_flipped_input[value_index][mixing_index] =+ float(input[token * DIMENSION + hadamard_index * Layout::MIXING_DIMENSION + mixing_index]) *+ rht_factors[hadamard_index * Layout::MIXING_DIMENSION + mixing_index];+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ typename Layout::RotatedColumns rotated;+ float local_maximum = 0.0f;+ uint output_mixing_index = 0;+ for (; output_mixing_index + Layout::COLUMNS_PER_PASS <= Layout::MIXING_DIMENSION;+ output_mixing_index += Layout::COLUMNS_PER_PASS) {+ rotate_columns<Layout, Layout::COLUMNS_PER_PASS>(+ output_mixing_index,+ sign_flipped_input,+ shared_mixing,+ scratch,+ rotated,+ local_maximum,+ lane,+ simdgroup+ );+ }+ if constexpr (Layout::MIXING_DIMENSION % Layout::COLUMNS_PER_PASS != 0) {+ rotate_columns<Layout, Layout::MIXING_DIMENSION % Layout::COLUMNS_PER_PASS>(+ output_mixing_index,+ sign_flipped_input,+ shared_mixing,+ scratch,+ rotated,+ local_maximum,+ lane,+ simdgroup+ );+ }++ const float maximum = reduce_activation_quantization_row_maximum<SIMDGROUPS_PER_THREADGROUP>(+ local_maximum,+ simdgroup_maxima,+ thread_context+ );+ const float scale = (isfinite(maximum) && maximum > 0.0f) ? precise::divide(maximum, 127.0f) : 1.0f;++ int local_column_group_sums[trellis::COLUMN_GROUP_COUNT] = {};+ METAL_PRAGMA_UNROLL+ for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {+ METAL_PRAGMA_UNROLL+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {+ const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);+ const uint column = hadamard_index * Layout::MIXING_DIMENSION + mixing_index;+ const int8_t quantized = quantize_activation_int8(rotated[mixing_index][value_index], scale);+ activations[token * DIMENSION + column] = quantized;+ METAL_PRAGMA_UNROLL+ for (uint column_group_index = 0; column_group_index < trellis::COLUMN_GROUP_COUNT; ++column_group_index) {+ local_column_group_sums[column_group_index] +=+ (column % trellis::COLUMN_GROUP_COUNT) == column_group_index ? int(quantized) : 0;+ }+ }+ }+ METAL_PRAGMA_UNROLL+ for (uint column_group_index = 0; column_group_index < trellis::COLUMN_GROUP_COUNT; ++column_group_index) {+ const int simdgroup_sum = simd_sum(local_column_group_sums[column_group_index]);+ if (lane == 0) {+ scratch[trellis::COLUMN_GROUP_COUNT * simdgroup + column_group_index] = float(simdgroup_sum);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (thread_index == 0) {+ float4 token_column_group_sums = float4(0.0f);+ for (uint source_simdgroup = 0; source_simdgroup < SIMDGROUPS_PER_THREADGROUP; ++source_simdgroup) {+ const uint first_group_sum_index = trellis::COLUMN_GROUP_COUNT * source_simdgroup;+ token_column_group_sums += float4(+ scratch[first_group_sum_index],+ scratch[first_group_sum_index + 1],+ scratch[first_group_sum_index + 2],+ scratch[first_group_sum_index + 3]+ );+ }+ column_group_sums[token] = token_column_group_sums;+ scales[token] = scale;+ }+}diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.h b/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.hindex 00f181402..b32304fff 100644--- a/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.h+++ b/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.h@@ -10,6 +10,10 @@ UZU_CONST uint ACTIVATION_QUANT_TILE_SIZE = 128;UZU_CONST float ACTIVATION_QUANT_INT8_MAX = 127.0f;#define ACTIVATION_QUANT_SIMDGROUPS 4+static METAL_FUNC int8_t quantize_activation_int8(const float value, const float scale) {+ return static_cast<int8_t>(clamp(round(value / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));+}+static METAL_FUNC uint nibble_grouped_index(const uint index) {constexpr uint NIBBLES_PER_BYTE = 2;constexpr uint CODES_PER_WORD = sizeof(uint) * NIBBLES_PER_BYTE;@@ -49,6 +53,23 @@ METAL_FUNC T reduce_activation_quantization_group(return result;}+template <uint SIMDGROUPS>+METAL_FUNC float reduce_activation_quantization_row_maximum(+ const float magnitude,+ threadgroup float* partials,+ const thread ThreadContext& thread_context+) {+ static_assert(SIMDGROUPS <= METAL_SIMD_SIZE, "one lane per simdgroup partial");+ const ushort lane_index = thread_context.simd_lane_id;+ const float simdgroup_maximum = simd_max(magnitude);+ if (lane_index == 0) {+ partials[thread_context.simdgroup_index] = simdgroup_maximum;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ // magnitudes are non-negative, so 0 is the identity of the maximum+ return simd_max(lane_index < SIMDGROUPS ? partials[lane_index] : 0.0f);+}+template <typename T>METAL_FUNC void write_activation_quantization_group(device T* output,diff --git a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metalindex b1a631046..19ab11278 100644--- a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal@@ -126,8 +126,7 @@ PUBLIC KERNEL(GatedActMul) ([](float x, float y) { return max(x, y); });const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;- const int8_t code =- static_cast<int8_t>(clamp(round(result / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));+ const int8_t code = quantize_activation_int8(result, scale);if (element_in_bounds) {const uint output_gated_index = grouped_by_weight_nibble ? nibble_grouped_index(gated_idx) : gated_idx;q_out[batch_idx * gated_dim + output_gated_index] = code;diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/trellis.h b/crates/uzu-engine/src/backends/metal/kernel/generated/trellis.hnew file mode 100644index 000000000..029ee080e--- /dev/null+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/trellis.h@@ -0,0 +1,9 @@+// Auto-generated from gpu_types/trellis - do not edit manually+#pragma once++#include <metal_stdlib>+using namespace metal;++namespace uzu::trellis {+static constant constexpr uint32_t COLUMN_GROUP_COUNT = 4;+} // namespace uzu::trellisdiff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rsindex 337413e9e..72471df67 100644--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs@@ -14,4 +14,5 @@ mod short_conv;mod ssm;mod tensor_add_bias_test;mod tensor_add_scale_test;+mod trellis_test;mod weaver;diff --git a/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rsnew file mode 100644index 000000000..d952d6b02--- /dev/null+++ b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs@@ -0,0 +1,104 @@+use half::bf16;+use rand::{RngExt, SeedableRng, rngs::SmallRng};+use rstest::rstest;+use uzu_engine_macros::uzu_test;++use crate::{+ backends::{+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,+ gpu_types::trellis::COLUMN_GROUP_COUNT,+ kernel::{TrellisTransformKernel, mixing_dimension},+ },+ cpu::Cpu,+ },+ tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},+};++const MIXING_ENTRY_BOUND: f32 = 0.6;+const INPUT_BASE_BOUND: f32 = 3.0;+const INPUT_OUTLIER_EXPONENT: i32 = 3;+const ALL_ZERO_TOKEN: usize = 1;+const BATCHES: [u32; 4] = [1, 3, 17, 65];++fn random_transform_data(+ rng: &mut SmallRng,+ columns: u32,+ batch: u32,+) -> (Vec<f32>, Vec<f32>, Vec<bf16>) {+ let mixing_dimension = mixing_dimension(columns);+ let rht_factors = (0..columns).map(|_| [1.0, -1.0][rng.random_range(0..2)]).collect();+ let mixing = (0..mixing_dimension * mixing_dimension)+ .map(|_| rng.random_range(-MIXING_ENTRY_BOUND..MIXING_ENTRY_BOUND))+ .collect();+ let input = (0..batch * columns)+ .map(|_| bf16::from_f32(rng.random_range(-INPUT_BASE_BOUND..INPUT_BASE_BOUND).powi(INPUT_OUTLIER_EXPONENT)))+ .collect();+ (rht_factors, mixing, input)+}++fn run<B: Backend>(+ columns: u32,+ batch: u32,+ rht_factors: &[f32],+ mixing: &[f32],+ input: &[bf16],+) -> (Vec<i8>, Vec<f32>, Vec<f32>) {+ let context = B::Context::new().expect("context");+ let transform =+ <B::Kernels as Kernels>::TrellisTransformKernel::new(context.as_ref(), columns).expect("trellis transform");+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");+ // kept alive until the command buffer has run: the CPU backend reads them at submit time+ let input = create_buffer_with_data::<B, bf16>(context.as_ref(), input);+ let rht_factors = create_buffer_with_data::<B, f32>(context.as_ref(), rht_factors);+ let mixing = create_buffer_with_data::<B, f32>(context.as_ref(), mixing);+ let mut activations = create_buffer::<B, i8>(context.as_ref(), (batch * columns) as usize);+ let mut column_group_sums = create_buffer::<B, f32>(context.as_ref(), COLUMN_GROUP_COUNT as usize * batch as usize);+ let mut scales = create_buffer::<B, f32>(context.as_ref(), batch as usize);+ transform.encode(+ &input,+ &rht_factors,+ &mixing,+ &mut activations,+ &mut column_group_sums,+ &mut scales,+ batch,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ (+ buffer_to_vec::<B, i8>(&activations),+ buffer_to_vec::<B, f32>(&column_group_sums),+ buffer_to_vec::<B, f32>(&scales),+ )+}++#[rstest]+#[test_attr(uzu_test)]+fn transform_matches_cpu(#[values(5120, 6144, 17408)] columns: u32) {+ let mut rng = SmallRng::seed_from_u64(u64::from(columns));+ for batch in BATCHES {+ let (rht_factors, mixing, mut input) = random_transform_data(&mut rng, columns, batch);+ if batch as usize > ALL_ZERO_TOKEN {+ input[ALL_ZERO_TOKEN * columns as usize..][..columns as usize].fill(bf16::ZERO);+ }+ let (expected_activations, _, expected_scales) = run::<Cpu>(columns, batch, &rht_factors, &mixing, &input);+ for_each_backend!(|B| {+ let (activations, column_group_sums, scales) = run::<B>(columns, batch, &rht_factors, &mixing, &input);+ let message = format!("{} columns {columns} batch {batch}", std::any::type_name::<B>());+ for (index, (&actual, &expected)) in activations.iter().zip(&expected_activations).enumerate() {+ assert!((i32::from(actual) - i32::from(expected)).abs() <= 1, "code {index}, {message}");+ }+ for (token, column_group_sums) in column_group_sums.chunks(COLUMN_GROUP_COUNT as usize).enumerate() {+ let mut sums_from_codes = [0.0f32; COLUMN_GROUP_COUNT as usize];+ for (column, &code) in activations[token * columns as usize..][..columns as usize].iter().enumerate() {+ sums_from_codes[column % COLUMN_GROUP_COUNT as usize] += f32::from(code);+ }+ assert_eq!(column_group_sums, sums_from_codes, "column group sums, token {token}, {message}");+ let expected_scale = expected_scales[token];+ let relative_error = (scales[token] - expected_scale).abs() / expected_scale.abs().max(1e-6);+ assert!(relative_error < 1e-3, "scale, token {token}, {message}");+ }+ });+ }+}