Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Trellis activation transform (#884)

CCYeh53ef316c0359
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
index a8184e988..628bd0dbe 100644
--- a/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
+++ b/crates/uzu-engine/src/backends/common/gpu_types/mod.rs
@@ -16,6 +16,7 @@ pub mod matmul;
pub mod quantization;
pub mod quantization_method;
pub mod ring;
+pub mod trellis;
pub mod trie;
pub mod weaver;
diff --git a/crates/uzu-engine/src/backends/common/gpu_types/trellis.rs b/crates/uzu-engine/src/backends/common/gpu_types/trellis.rs
new file mode 100644
index 000000000..f2d7282fb
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/gpu_types/trellis.rs
@@ -0,0 +1 @@
+pub const COLUMN_GROUP_COUNT: u32 = 4;
diff --git a/crates/uzu-engine/src/backends/common/kernel/mod.rs b/crates/uzu-engine/src/backends/common/kernel/mod.rs
index 3b665c52a..bc85140b3 100644
--- a/crates/uzu-engine/src/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/mod.rs
@@ -9,10 +9,12 @@ pub mod delta_net_tree_verify;
pub mod gated_act_mul;
pub mod matmul;
pub mod radix_top_k_small;
+mod trellis;
pub use activation_transform::{ActivationQuantization, ActivationTransform};
pub use attention::{AttentionArguments, AttentionKernel, AttentionKernelConfig};
pub use gated_act_mul::{GatedActMul, GatedActMulSettings};
+pub use trellis::mixing_dimension;
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
diff --git a/crates/uzu-engine/src/backends/common/kernel/trellis.rs b/crates/uzu-engine/src/backends/common/kernel/trellis.rs
new file mode 100644
index 000000000..1b3f1e4b6
--- /dev/null
+++ b/crates/uzu-engine/src/backends/common/kernel/trellis.rs
@@ -0,0 +1,4 @@
+/// Mixing matrix side length: columns divided by their largest power-of-two factor
+pub fn mixing_dimension(columns: u32) -> u32 {
+ columns >> columns.trailing_zeros()
+}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rs
index c6e0ce5ae..28b1bfe8c 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/mod.rs
@@ -1,6 +1,5 @@
pub mod activation_transform;
-
-use crate::backends::common::gpu_types::HADAMARD_TRANSFORM_BLOCK_SIZE;
+pub(super) mod trellis;
pub const INT8_SYMMETRIC_QUANTIZATION_MAXIMUM: f32 = 127.0;
@@ -24,10 +23,10 @@ pub fn quantize_symmetric_i8(
(value / divisor).round().clamp(-INT8_SYMMETRIC_QUANTIZATION_MAXIMUM, INT8_SYMMETRIC_QUANTIZATION_MAXIMUM) as i8
}
-pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize]) {
+pub fn hadamard_transform(values: &mut [f32]) {
let mut stride = 1;
- while stride < HADAMARD_TRANSFORM_BLOCK_SIZE as usize {
- for lane in 0..HADAMARD_TRANSFORM_BLOCK_SIZE as usize {
+ while stride < values.len() {
+ for lane in 0..values.len() {
if lane & stride == 0 {
let a = values[lane];
let b = values[lane | stride];
@@ -37,7 +36,7 @@ pub(crate) fn hadamard_transform(values: &mut [f32; HADAMARD_TRANSFORM_BLOCK_SIZ
}
stride <<= 1;
}
- let scale = 1.0 / (HADAMARD_TRANSFORM_BLOCK_SIZE as f32).sqrt();
+ let scale = 1.0 / (values.len() as f32).sqrt();
for v in values.iter_mut() {
*v *= scale;
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs
new file mode 100644
index 000000000..3455d9099
--- /dev/null
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs
@@ -0,0 +1,71 @@
+use half::bf16;
+use uzu_engine_macros::kernel;
+
+use super::{hadamard_transform, min_max_symmetric_divisor, quantize_symmetric_i8};
+use crate::backends::common::{gpu_types::trellis::COLUMN_GROUP_COUNT, kernel::mixing_dimension};
+
+fn rotate_token(
+ input: &[bf16],
+ rht_factors: &[f32],
+ mixing: &[f32],
+ mixing_dimension: usize,
+) -> Vec<f32> {
+ let columns = input.len();
+ let hadamard_size = columns / mixing_dimension;
+ let mut rotated = vec![0.0f32; columns];
+ for output_mixing_index in 0..mixing_dimension {
+ let mut hadamard_values: Vec<f32> = (0..hadamard_size)
+ .map(|hadamard_index| {
+ (0..mixing_dimension).fold(0.0f32, |accumulated, mixing_index| {
+ let column = hadamard_index * mixing_dimension + mixing_index;
+ (input[column].to_f32() * rht_factors[column])
+ .mul_add(mixing[output_mixing_index * mixing_dimension + mixing_index], accumulated)
+ })
+ })
+ .collect();
+ hadamard_transform(&mut hadamard_values);
+ for (hadamard_index, value) in hadamard_values.into_iter().enumerate() {
+ rotated[hadamard_index * mixing_dimension + output_mixing_index] = bf16::from_f32(value).to_f32();
+ }
+ }
+ rotated
+}
+
+#[kernel(TrellisTransform)]
+#[variants(DIMENSION, 5120, 6144, 17408)]
+pub fn trellis_transform<const DIMENSION: u32>(
+ input: *const bf16,
+ rht_factors: *const f32,
+ mixing: *const f32,
+ activations: *mut i8,
+ column_group_sums: *mut f32,
+ scales: *mut f32,
+ batch: u32,
+) {
+ let columns = DIMENSION as usize;
+ let mixing_dimension = mixing_dimension(DIMENSION) as usize;
+ let rht_factors = unsafe { std::slice::from_raw_parts(rht_factors, columns) };
+ let mixing = unsafe { std::slice::from_raw_parts(mixing, mixing_dimension * mixing_dimension) };
+
+ for token in 0..batch as usize {
+ let row = unsafe { std::slice::from_raw_parts(input.add(token * columns), columns) };
+ let quantized_row = unsafe { std::slice::from_raw_parts_mut(activations.add(token * columns), columns) };
+ let column_group_sums_row = unsafe {
+ std::slice::from_raw_parts_mut(
+ column_group_sums.add(COLUMN_GROUP_COUNT as usize * token),
+ COLUMN_GROUP_COUNT as usize,
+ )
+ };
+
+ let rotated = rotate_token(row, rht_factors, mixing, mixing_dimension);
+ let scale = min_max_symmetric_divisor(&rotated);
+ let mut token_column_group_sums = [0i32; COLUMN_GROUP_COUNT as usize];
+ for (column, (quantized, &value)) in quantized_row.iter_mut().zip(&rotated).enumerate() {
+ *quantized = quantize_symmetric_i8(value, scale);
+ token_column_group_sums[column % COLUMN_GROUP_COUNT as usize] += i32::from(*quantized);
+ }
+
+ column_group_sums_row.copy_from_slice(&token_column_group_sums.map(|sum| sum as f32));
+ unsafe { *scales.add(token) = scale };
+ }
+}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
index c4956c05f..16399ee54 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/normalization/normalization.rs
@@ -126,7 +126,7 @@ pub fn normalization<
// Input RHT like the Metal kernel: apply the factors, then transform each block, before output scaling
if let Some(factors) = hadamard_factors {
for block_start in (0..element_count).step_by(HADAMARD_TRANSFORM_BLOCK_SIZE as usize) {
- let mut block = std::array::from_fn(|lane| {
+ let mut block: [f32; HADAMARD_TRANSFORM_BLOCK_SIZE as usize] = std::array::from_fn(|lane| {
let factor = unsafe { *factors.add(block_start + lane) } as f32;
row[block_start + lane].to_f32().unwrap() * factor
});
diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
index a23827380..b4df7bf5d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/activation_transform.metal
@@ -81,9 +81,7 @@ PUBLIC KERNEL(ActivationTransform)(
[](float x, float y) { return max(x, y); }
);
const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;
-
- const int8_t code =
- static_cast<int8_t>(clamp(round(value / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));
+ const int8_t code = quantize_activation_int8(value, scale);
if (in_bounds) {
const uint output_factor_index = grouped_by_weight_nibble ? nibble_grouped_index(factor_index) : factor_index;
q_out[batch_index * element_count + output_factor_index] = code;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal
new file mode 100644
index 000000000..0ea879905
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal
@@ -0,0 +1,259 @@
+#include <metal_stdlib>
+#include "../common/activation_quantization.h"
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../generated/trellis.h"
+
+using namespace metal;
+namespace trellis = uzu::trellis;
+
+#define THREADS_PER_THREADGROUP 512
+
+UZU_CONST uint SIMDGROUPS_PER_THREADGROUP = THREADS_PER_THREADGROUP / METAL_SIMD_SIZE;
+
+UZU_CONST uint MAX_MIXING_DIMENSION = 17;
+
+static METAL_FUNC float butterfly(float value, ushort lane, ushort lane_stride) {
+ const float other = simd_shuffle_xor(value, lane_stride);
+ return (lane & lane_stride) != 0 ? other - value : value + other;
+}
+
+static METAL_FUNC uint bank_conflict_free_offset(uint hadamard_index) {
+ return hadamard_index + hadamard_index / METAL_SIMD_SIZE;
+}
+
+static METAL_FUNC uint hadamard_index_before_transpose(ushort lane, ushort simdgroup, uint value_index) {
+ return METAL_SIMD_SIZE * (simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index) + lane;
+}
+
+template <uint HADAMARD_SIZE>
+static METAL_FUNC uint hadamard_index_after_transpose(ushort lane, ushort simdgroup, uint value_index) {
+ if (HADAMARD_SIZE == 1024) {
+ return METAL_SIMD_SIZE * lane + simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index;
+ }
+ return METAL_SIMD_SIZE * (lane + METAL_SIMD_SIZE * (value_index % 2)) + simdgroup +
+ SIMDGROUPS_PER_THREADGROUP * (value_index / 2);
+}
+
+template <uint DIMENSION>
+struct TransformLayout {
+ static constant constexpr uint HADAMARD_SIZE = DIMENSION & (0u - DIMENSION);
+ static constant constexpr uint MIXING_DIMENSION = DIMENSION / HADAMARD_SIZE;
+ static constant constexpr uint VALUES_PER_THREAD = HADAMARD_SIZE / THREADS_PER_THREADGROUP;
+ static constant constexpr uint COLUMNS_PER_PASS = HADAMARD_SIZE == 2048 ? 1 : (MIXING_DIMENSION > 8 ? 4 : 2);
+ static constant constexpr uint COLUMN_SCRATCH_SIZE = HADAMARD_SIZE + HADAMARD_SIZE / METAL_SIMD_SIZE;
+ static constant constexpr uint SCRATCH_SIZE = COLUMNS_PER_PASS * COLUMN_SCRATCH_SIZE;
+ typedef float SignFlippedInput[VALUES_PER_THREAD][MIXING_DIMENSION];
+ typedef float RotatedColumns[MIXING_DIMENSION][VALUES_PER_THREAD];
+};
+
+template <typename Layout, uint COLUMNS>
+static METAL_FUNC void hadamard_transform_lanes(
+ thread float (&column_values)[COLUMNS][Layout::VALUES_PER_THREAD],
+ const ushort lane
+) {
+ METAL_PRAGMA_UNROLL
+ for (ushort lane_stride = 1; lane_stride <= METAL_SIMD_SIZE / 2; lane_stride <<= 1) {
+ METAL_PRAGMA_UNROLL
+ for (uint column = 0; column < COLUMNS; ++column) {
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
+ column_values[column][value_index] = butterfly(column_values[column][value_index], lane, lane_stride);
+ }
+ }
+ }
+}
+
+template <typename Layout, uint COLUMNS>
+static METAL_FUNC void rotate_columns(
+ const uint first_output_mixing_index,
+ const thread typename Layout::SignFlippedInput& sign_flipped_input,
+ const threadgroup float* shared_mixing,
+ threadgroup float* scratch,
+ thread typename Layout::RotatedColumns& rotated,
+ thread float& local_maximum,
+ const ushort lane,
+ const ushort simdgroup
+) {
+ const float normalization = 1.0f / sqrt(float(Layout::HADAMARD_SIZE));
+
+ float column_values[COLUMNS][Layout::VALUES_PER_THREAD] = {};
+ METAL_PRAGMA_UNROLL
+ for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {
+ METAL_PRAGMA_UNROLL
+ for (uint column = 0; column < COLUMNS; ++column) {
+ const float mixing =
+ shared_mixing[(first_output_mixing_index + column) * Layout::MIXING_DIMENSION + mixing_index];
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
+ column_values[column][value_index] =
+ fma(sign_flipped_input[value_index][mixing_index], mixing, column_values[column][value_index]);
+ }
+ }
+ }
+ hadamard_transform_lanes<Layout, COLUMNS>(column_values, lane);
+ METAL_PRAGMA_UNROLL
+ for (uint column = 0; column < COLUMNS; ++column) {
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
+ scratch
+ [column * Layout::COLUMN_SCRATCH_SIZE +
+ bank_conflict_free_offset(hadamard_index_before_transpose(lane, simdgroup, value_index))] =
+ column_values[column][value_index];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ METAL_PRAGMA_UNROLL
+ for (uint column = 0; column < COLUMNS; ++column) {
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
+ const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);
+ column_values[column][value_index] =
+ scratch[column * Layout::COLUMN_SCRATCH_SIZE + bank_conflict_free_offset(hadamard_index)];
+ }
+ }
+ hadamard_transform_lanes<Layout, COLUMNS>(column_values, lane);
+ if constexpr (Layout::HADAMARD_SIZE == 2048) {
+ METAL_PRAGMA_UNROLL
+ for (uint column = 0; column < COLUMNS; ++column) {
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; value_index += 2) {
+ const float lower = column_values[column][value_index];
+ const float upper = column_values[column][value_index + 1];
+ column_values[column][value_index] = lower + upper;
+ column_values[column][value_index + 1] = lower - upper;
+ }
+ }
+ }
+ METAL_PRAGMA_UNROLL
+ for (uint column = 0; column < COLUMNS; ++column) {
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
+ rotated[first_output_mixing_index + column][value_index] =
+ float(bfloat(column_values[column][value_index] * normalization));
+ local_maximum = max(local_maximum, abs(rotated[first_output_mixing_index + column][value_index]));
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+}
+
+// TODO: a columns agnostic kernel
+template <uint DIMENSION>
+VARIANTS(DIMENSION, 5120, 6144, 17408)
+PUBLIC KERNEL(TrellisTransform)(
+ device const bfloat* input,
+ device const float* rht_factors,
+ device const float* mixing,
+ device int8_t* activations,
+ device float4* column_group_sums,
+ device float* scales,
+ constant uint& batch,
+ threadgroup float scratch[TransformLayout<DIMENSION>::SCRATCH_SIZE],
+ threadgroup float simdgroup_maxima[SIMDGROUPS_PER_THREADGROUP],
+ threadgroup float shared_mixing[MAX_MIXING_DIMENSION * MAX_MIXING_DIMENSION],
+ const uint token GROUPS(batch),
+ const uint thread_index THREADS(THREADS_PER_THREADGROUP),
+ const ThreadContext thread_context
+) {
+ using Layout = TransformLayout<DIMENSION>;
+ static_assert(Layout::HADAMARD_SIZE == 1024 || Layout::HADAMARD_SIZE == 2048, "unsupported Hadamard size");
+ static_assert(Layout::MIXING_DIMENSION <= MAX_MIXING_DIMENSION, "mixing matrix does not fit threadgroup memory");
+
+ const ushort lane = ushort(thread_context.simd_lane_id);
+ const ushort simdgroup = ushort(thread_context.simdgroup_index);
+
+ for (uint index = thread_index; index < Layout::MIXING_DIMENSION * Layout::MIXING_DIMENSION;
+ index += THREADS_PER_THREADGROUP) {
+ shared_mixing[index] = mixing[index];
+ }
+
+ typename Layout::SignFlippedInput sign_flipped_input;
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
+ const uint hadamard_index = hadamard_index_before_transpose(lane, simdgroup, value_index);
+ METAL_PRAGMA_UNROLL
+ for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {
+ sign_flipped_input[value_index][mixing_index] =
+ float(input[token * DIMENSION + hadamard_index * Layout::MIXING_DIMENSION + mixing_index]) *
+ rht_factors[hadamard_index * Layout::MIXING_DIMENSION + mixing_index];
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ typename Layout::RotatedColumns rotated;
+ float local_maximum = 0.0f;
+ uint output_mixing_index = 0;
+ for (; output_mixing_index + Layout::COLUMNS_PER_PASS <= Layout::MIXING_DIMENSION;
+ output_mixing_index += Layout::COLUMNS_PER_PASS) {
+ rotate_columns<Layout, Layout::COLUMNS_PER_PASS>(
+ output_mixing_index,
+ sign_flipped_input,
+ shared_mixing,
+ scratch,
+ rotated,
+ local_maximum,
+ lane,
+ simdgroup
+ );
+ }
+ if constexpr (Layout::MIXING_DIMENSION % Layout::COLUMNS_PER_PASS != 0) {
+ rotate_columns<Layout, Layout::MIXING_DIMENSION % Layout::COLUMNS_PER_PASS>(
+ output_mixing_index,
+ sign_flipped_input,
+ shared_mixing,
+ scratch,
+ rotated,
+ local_maximum,
+ lane,
+ simdgroup
+ );
+ }
+
+ const float maximum = reduce_activation_quantization_row_maximum<SIMDGROUPS_PER_THREADGROUP>(
+ local_maximum,
+ simdgroup_maxima,
+ thread_context
+ );
+ const float scale = (isfinite(maximum) && maximum > 0.0f) ? precise::divide(maximum, 127.0f) : 1.0f;
+
+ int local_column_group_sums[trellis::COLUMN_GROUP_COUNT] = {};
+ METAL_PRAGMA_UNROLL
+ for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
+ const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);
+ const uint column = hadamard_index * Layout::MIXING_DIMENSION + mixing_index;
+ const int8_t quantized = quantize_activation_int8(rotated[mixing_index][value_index], scale);
+ activations[token * DIMENSION + column] = quantized;
+ METAL_PRAGMA_UNROLL
+ for (uint column_group_index = 0; column_group_index < trellis::COLUMN_GROUP_COUNT; ++column_group_index) {
+ local_column_group_sums[column_group_index] +=
+ (column % trellis::COLUMN_GROUP_COUNT) == column_group_index ? int(quantized) : 0;
+ }
+ }
+ }
+ METAL_PRAGMA_UNROLL
+ for (uint column_group_index = 0; column_group_index < trellis::COLUMN_GROUP_COUNT; ++column_group_index) {
+ const int simdgroup_sum = simd_sum(local_column_group_sums[column_group_index]);
+ if (lane == 0) {
+ scratch[trellis::COLUMN_GROUP_COUNT * simdgroup + column_group_index] = float(simdgroup_sum);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (thread_index == 0) {
+ float4 token_column_group_sums = float4(0.0f);
+ for (uint source_simdgroup = 0; source_simdgroup < SIMDGROUPS_PER_THREADGROUP; ++source_simdgroup) {
+ const uint first_group_sum_index = trellis::COLUMN_GROUP_COUNT * source_simdgroup;
+ token_column_group_sums += float4(
+ scratch[first_group_sum_index],
+ scratch[first_group_sum_index + 1],
+ scratch[first_group_sum_index + 2],
+ scratch[first_group_sum_index + 3]
+ );
+ }
+ column_group_sums[token] = token_column_group_sums;
+ scales[token] = scale;
+ }
+}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.h b/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.h
index 00f181402..b32304fff 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/common/activation_quantization.h
@@ -10,6 +10,10 @@ UZU_CONST uint ACTIVATION_QUANT_TILE_SIZE = 128;
UZU_CONST float ACTIVATION_QUANT_INT8_MAX = 127.0f;
#define ACTIVATION_QUANT_SIMDGROUPS 4
+static METAL_FUNC int8_t quantize_activation_int8(const float value, const float scale) {
+ return static_cast<int8_t>(clamp(round(value / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));
+}
+
static METAL_FUNC uint nibble_grouped_index(const uint index) {
constexpr uint NIBBLES_PER_BYTE = 2;
constexpr uint CODES_PER_WORD = sizeof(uint) * NIBBLES_PER_BYTE;
@@ -49,6 +53,23 @@ METAL_FUNC T reduce_activation_quantization_group(
return result;
}
+template <uint SIMDGROUPS>
+METAL_FUNC float reduce_activation_quantization_row_maximum(
+ const float magnitude,
+ threadgroup float* partials,
+ const thread ThreadContext& thread_context
+) {
+ static_assert(SIMDGROUPS <= METAL_SIMD_SIZE, "one lane per simdgroup partial");
+ const ushort lane_index = thread_context.simd_lane_id;
+ const float simdgroup_maximum = simd_max(magnitude);
+ if (lane_index == 0) {
+ partials[thread_context.simdgroup_index] = simdgroup_maximum;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ // magnitudes are non-negative, so 0 is the identity of the maximum
+ return simd_max(lane_index < SIMDGROUPS ? partials[lane_index] : 0.0f);
+}
+
template <typename T>
METAL_FUNC void write_activation_quantization_group(
device T* output,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
index b1a631046..19ab11278 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal
@@ -126,8 +126,7 @@ PUBLIC KERNEL(GatedActMul) (
[](float x, float y) { return max(x, y); }
);
const float scale = isfinite(maximum) && maximum > 0.0f ? maximum / ACTIVATION_QUANT_INT8_MAX : 1.0f;
- const int8_t code =
- static_cast<int8_t>(clamp(round(result / scale), -ACTIVATION_QUANT_INT8_MAX, ACTIVATION_QUANT_INT8_MAX));
+ const int8_t code = quantize_activation_int8(result, scale);
if (element_in_bounds) {
const uint output_gated_index = grouped_by_weight_nibble ? nibble_grouped_index(gated_idx) : gated_idx;
q_out[batch_idx * gated_dim + output_gated_index] = code;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/generated/trellis.h b/crates/uzu-engine/src/backends/metal/kernel/generated/trellis.h
new file mode 100644
index 000000000..029ee080e
--- /dev/null
+++ b/crates/uzu-engine/src/backends/metal/kernel/generated/trellis.h
@@ -0,0 +1,9 @@
+// Auto-generated from gpu_types/trellis - do not edit manually
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu::trellis {
+static constant constexpr uint32_t COLUMN_GROUP_COUNT = 4;
+} // namespace uzu::trellis
diff --git a/crates/uzu-engine/unit/backends/common/kernel/mod.rs b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
index 337413e9e..72471df67 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/mod.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/mod.rs
@@ -14,4 +14,5 @@ mod short_conv;
mod ssm;
mod tensor_add_bias_test;
mod tensor_add_scale_test;
+mod trellis_test;
mod weaver;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs
new file mode 100644
index 000000000..d952d6b02
--- /dev/null
+++ b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs
@@ -0,0 +1,104 @@
+use half::bf16;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+use rstest::rstest;
+use uzu_engine_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context, Kernels,
+ gpu_types::trellis::COLUMN_GROUP_COUNT,
+ kernel::{TrellisTransformKernel, mixing_dimension},
+ },
+ cpu::Cpu,
+ },
+ tests::helpers::{buffer_to_vec, create_buffer, create_buffer_with_data, for_each_backend},
+};
+
+const MIXING_ENTRY_BOUND: f32 = 0.6;
+const INPUT_BASE_BOUND: f32 = 3.0;
+const INPUT_OUTLIER_EXPONENT: i32 = 3;
+const ALL_ZERO_TOKEN: usize = 1;
+const BATCHES: [u32; 4] = [1, 3, 17, 65];
+
+fn random_transform_data(
+ rng: &mut SmallRng,
+ columns: u32,
+ batch: u32,
+) -> (Vec<f32>, Vec<f32>, Vec<bf16>) {
+ let mixing_dimension = mixing_dimension(columns);
+ let rht_factors = (0..columns).map(|_| [1.0, -1.0][rng.random_range(0..2)]).collect();
+ let mixing = (0..mixing_dimension * mixing_dimension)
+ .map(|_| rng.random_range(-MIXING_ENTRY_BOUND..MIXING_ENTRY_BOUND))
+ .collect();
+ let input = (0..batch * columns)
+ .map(|_| bf16::from_f32(rng.random_range(-INPUT_BASE_BOUND..INPUT_BASE_BOUND).powi(INPUT_OUTLIER_EXPONENT)))
+ .collect();
+ (rht_factors, mixing, input)
+}
+
+fn run<B: Backend>(
+ columns: u32,
+ batch: u32,
+ rht_factors: &[f32],
+ mixing: &[f32],
+ input: &[bf16],
+) -> (Vec<i8>, Vec<f32>, Vec<f32>) {
+ let context = B::Context::new().expect("context");
+ let transform =
+ <B::Kernels as Kernels>::TrellisTransformKernel::new(context.as_ref(), columns).expect("trellis transform");
+ let mut command_buffer = context.create_command_buffer(None, None).expect("command buffer");
+ // kept alive until the command buffer has run: the CPU backend reads them at submit time
+ let input = create_buffer_with_data::<B, bf16>(context.as_ref(), input);
+ let rht_factors = create_buffer_with_data::<B, f32>(context.as_ref(), rht_factors);
+ let mixing = create_buffer_with_data::<B, f32>(context.as_ref(), mixing);
+ let mut activations = create_buffer::<B, i8>(context.as_ref(), (batch * columns) as usize);
+ let mut column_group_sums = create_buffer::<B, f32>(context.as_ref(), COLUMN_GROUP_COUNT as usize * batch as usize);
+ let mut scales = create_buffer::<B, f32>(context.as_ref(), batch as usize);
+ transform.encode(
+ &input,
+ &rht_factors,
+ &mixing,
+ &mut activations,
+ &mut column_group_sums,
+ &mut scales,
+ batch,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ (
+ buffer_to_vec::<B, i8>(&activations),
+ buffer_to_vec::<B, f32>(&column_group_sums),
+ buffer_to_vec::<B, f32>(&scales),
+ )
+}
+
+#[rstest]
+#[test_attr(uzu_test)]
+fn transform_matches_cpu(#[values(5120, 6144, 17408)] columns: u32) {
+ let mut rng = SmallRng::seed_from_u64(u64::from(columns));
+ for batch in BATCHES {
+ let (rht_factors, mixing, mut input) = random_transform_data(&mut rng, columns, batch);
+ if batch as usize > ALL_ZERO_TOKEN {
+ input[ALL_ZERO_TOKEN * columns as usize..][..columns as usize].fill(bf16::ZERO);
+ }
+ let (expected_activations, _, expected_scales) = run::<Cpu>(columns, batch, &rht_factors, &mixing, &input);
+ for_each_backend!(|B| {
+ let (activations, column_group_sums, scales) = run::<B>(columns, batch, &rht_factors, &mixing, &input);
+ let message = format!("{} columns {columns} batch {batch}", std::any::type_name::<B>());
+ for (index, (&actual, &expected)) in activations.iter().zip(&expected_activations).enumerate() {
+ assert!((i32::from(actual) - i32::from(expected)).abs() <= 1, "code {index}, {message}");
+ }
+ for (token, column_group_sums) in column_group_sums.chunks(COLUMN_GROUP_COUNT as usize).enumerate() {
+ let mut sums_from_codes = [0.0f32; COLUMN_GROUP_COUNT as usize];
+ for (column, &code) in activations[token * columns as usize..][..columns as usize].iter().enumerate() {
+ sums_from_codes[column % COLUMN_GROUP_COUNT as usize] += f32::from(code);
+ }
+ assert_eq!(column_group_sums, sums_from_codes, "column group sums, token {token}, {message}");
+ let expected_scale = expected_scales[token];
+ let relative_error = (scales[token] - expected_scale).abs() / expected_scale.abs().max(1e-6);
+ assert!(relative_error < 1e-3, "scale, token {token}, {message}");
+ }
+ });
+ }
+}