A high-performance inference engine for AI models
trellis transform for muse glimmer (#902)
CCYeh615c09605f8d
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rsindex 3455d9099..d8e9685f0 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs@@ -32,7 +32,7 @@ fn rotate_token(}#[kernel(TrellisTransform)]-#[variants(DIMENSION, 5120, 6144, 17408)]+#[variants(DIMENSION, 4096, 5120, 6144, 6656, 17408, 19968)]pub fn trellis_transform<const DIMENSION: u32>(input: *const bf16,rht_factors: *const f32,diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metalindex 0ea879905..a365e1f68 100644--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal@@ -12,7 +12,7 @@ namespace trellis = uzu::trellis;UZU_CONST uint SIMDGROUPS_PER_THREADGROUP = THREADS_PER_THREADGROUP / METAL_SIMD_SIZE;-UZU_CONST uint MAX_MIXING_DIMENSION = 17;+UZU_CONST uint THREADGROUP_MEMORY_LIMIT = 32768;static METAL_FUNC float butterfly(float value, ushort lane, ushort lane_stride) {const float other = simd_shuffle_xor(value, lane_stride);@@ -27,34 +27,42 @@ static METAL_FUNC uint hadamard_index_before_transpose(ushort lane, ushort simdgreturn METAL_SIMD_SIZE * (simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index) + lane;}-template <uint HADAMARD_SIZE>-static METAL_FUNC uint hadamard_index_after_transpose(ushort lane, ushort simdgroup, uint value_index) {- if (HADAMARD_SIZE == 1024) {- return METAL_SIMD_SIZE * lane + simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index;- }- return METAL_SIMD_SIZE * (lane + METAL_SIMD_SIZE * (value_index % 2)) + simdgroup +- SIMDGROUPS_PER_THREADGROUP * (value_index / 2);-}-template <uint DIMENSION>struct TransformLayout {static constant constexpr uint HADAMARD_SIZE = DIMENSION & (0u - DIMENSION);static constant constexpr uint MIXING_DIMENSION = DIMENSION / HADAMARD_SIZE;static constant constexpr uint VALUES_PER_THREAD = HADAMARD_SIZE / THREADS_PER_THREADGROUP;- static constant constexpr uint COLUMNS_PER_PASS = HADAMARD_SIZE == 2048 ? 1 : (MIXING_DIMENSION > 8 ? 4 : 2);+ static constant constexpr uint IN_THREAD_VALUES = HADAMARD_SIZE > 1024 ? HADAMARD_SIZE / 1024 : 1;+ static constant constexpr uint SECOND_LANE_STRIDE_LIMIT =+ HADAMARD_SIZE == THREADS_PER_THREADGROUP ? METAL_SIMD_SIZE / 4 : METAL_SIMD_SIZE / 2;+ static constant constexpr uint COLUMNS_PER_PASS = HADAMARD_SIZE >= 2048 ? 1 : (MIXING_DIMENSION > 8 ? 4 : 2);static constant constexpr uint COLUMN_SCRATCH_SIZE = HADAMARD_SIZE + HADAMARD_SIZE / METAL_SIMD_SIZE;static constant constexpr uint SCRATCH_SIZE = COLUMNS_PER_PASS * COLUMN_SCRATCH_SIZE;+ static constant constexpr uint MIXING_SIZE = MIXING_DIMENSION * MIXING_DIMENSION;+ static constant constexpr uint THREADGROUP_BYTES =+ sizeof(float) * (SCRATCH_SIZE + SIMDGROUPS_PER_THREADGROUP + MIXING_SIZE);typedef float SignFlippedInput[VALUES_PER_THREAD][MIXING_DIMENSION];typedef float RotatedColumns[MIXING_DIMENSION][VALUES_PER_THREAD];};-template <typename Layout, uint COLUMNS>+template <typename Layout>+static METAL_FUNC uint hadamard_index_after_transpose(ushort lane, ushort simdgroup, uint value_index) {+ if constexpr (Layout::HADAMARD_SIZE == THREADS_PER_THREADGROUP) {+ constexpr uint half_simd = METAL_SIMD_SIZE / 2;+ return METAL_SIMD_SIZE * (lane % half_simd) + half_simd * (lane / half_simd) + simdgroup;+ } else {+ return METAL_SIMD_SIZE * (lane + METAL_SIMD_SIZE * (value_index % Layout::IN_THREAD_VALUES)) + simdgroup ++ SIMDGROUPS_PER_THREADGROUP * (value_index / Layout::IN_THREAD_VALUES);+ }+}++template <typename Layout, uint COLUMNS, uint LANE_STRIDE_LIMIT = METAL_SIMD_SIZE / 2>static METAL_FUNC void hadamard_transform_lanes(thread float (&column_values)[COLUMNS][Layout::VALUES_PER_THREAD],const ushort lane) {METAL_PRAGMA_UNROLL- for (ushort lane_stride = 1; lane_stride <= METAL_SIMD_SIZE / 2; lane_stride <<= 1) {+ for (ushort lane_stride = 1; lane_stride <= LANE_STRIDE_LIMIT; lane_stride <<= 1) {METAL_PRAGMA_UNROLLfor (uint column = 0; column < COLUMNS; ++column) {METAL_PRAGMA_UNROLL@@ -109,21 +117,25 @@ static METAL_FUNC void rotate_columns(for (uint column = 0; column < COLUMNS; ++column) {METAL_PRAGMA_UNROLLfor (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {- const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);+ const uint hadamard_index = hadamard_index_after_transpose<Layout>(lane, simdgroup, value_index);column_values[column][value_index] =scratch[column * Layout::COLUMN_SCRATCH_SIZE + bank_conflict_free_offset(hadamard_index)];}}- hadamard_transform_lanes<Layout, COLUMNS>(column_values, lane);- if constexpr (Layout::HADAMARD_SIZE == 2048) {+ hadamard_transform_lanes<Layout, COLUMNS, Layout::SECOND_LANE_STRIDE_LIMIT>(column_values, lane);+ METAL_PRAGMA_UNROLL+ for (uint value_stride = 1; value_stride < Layout::IN_THREAD_VALUES; value_stride <<= 1) {METAL_PRAGMA_UNROLLfor (uint column = 0; column < COLUMNS; ++column) {METAL_PRAGMA_UNROLL- for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; value_index += 2) {- const float lower = column_values[column][value_index];- const float upper = column_values[column][value_index + 1];- column_values[column][value_index] = lower + upper;- column_values[column][value_index + 1] = lower - upper;+ for (uint block = 0; block < Layout::VALUES_PER_THREAD; block += 2 * value_stride) {+ METAL_PRAGMA_UNROLL+ for (uint value_index = block; value_index < block + value_stride; ++value_index) {+ const float lower = column_values[column][value_index];+ const float upper = column_values[column][value_index + value_stride];+ column_values[column][value_index] = lower + upper;+ column_values[column][value_index + value_stride] = lower - upper;+ }}}}@@ -141,7 +153,7 @@ static METAL_FUNC void rotate_columns(// TODO: a columns agnostic kerneltemplate <uint DIMENSION>-VARIANTS(DIMENSION, 5120, 6144, 17408)+VARIANTS(DIMENSION, 4096, 5120, 6144, 6656, 17408, 19968)PUBLIC KERNEL(TrellisTransform)(device const bfloat* input,device const float* rht_factors,@@ -152,14 +164,17 @@ PUBLIC KERNEL(TrellisTransform)(constant uint& batch,threadgroup float scratch[TransformLayout<DIMENSION>::SCRATCH_SIZE],threadgroup float simdgroup_maxima[SIMDGROUPS_PER_THREADGROUP],- threadgroup float shared_mixing[MAX_MIXING_DIMENSION * MAX_MIXING_DIMENSION],+ threadgroup float shared_mixing[TransformLayout<DIMENSION>::MIXING_SIZE],const uint token GROUPS(batch),const uint thread_index THREADS(THREADS_PER_THREADGROUP),const ThreadContext thread_context) {using Layout = TransformLayout<DIMENSION>;- static_assert(Layout::HADAMARD_SIZE == 1024 || Layout::HADAMARD_SIZE == 2048, "unsupported Hadamard size");- static_assert(Layout::MIXING_DIMENSION <= MAX_MIXING_DIMENSION, "mixing matrix does not fit threadgroup memory");+ static_assert(+ Layout::HADAMARD_SIZE >= THREADS_PER_THREADGROUP && Layout::HADAMARD_SIZE <= 4096,+ "unsupported Hadamard size"+ );+ static_assert(Layout::THREADGROUP_BYTES <= THREADGROUP_MEMORY_LIMIT, "does not fit threadgroup memory");const ushort lane = ushort(thread_context.simd_lane_id);const ushort simdgroup = ushort(thread_context.simdgroup_index);@@ -223,7 +238,7 @@ PUBLIC KERNEL(TrellisTransform)(for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {METAL_PRAGMA_UNROLLfor (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {- const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);+ const uint hadamard_index = hadamard_index_after_transpose<Layout>(lane, simdgroup, value_index);const uint column = hadamard_index * Layout::MIXING_DIMENSION + mixing_index;const int8_t quantized = quantize_activation_int8(rotated[mixing_index][value_index], scale);activations[token * DIMENSION + column] = quantized;diff --git a/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rsindex d952d6b02..7c08d4716 100644--- a/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs@@ -19,7 +19,7 @@ const MIXING_ENTRY_BOUND: f32 = 0.6;const INPUT_BASE_BOUND: f32 = 3.0;const INPUT_OUTLIER_EXPONENT: i32 = 3;const ALL_ZERO_TOKEN: usize = 1;-const BATCHES: [u32; 4] = [1, 3, 17, 65];+const BATCHES: [u32; 5] = [1, 8, 16, 32, 1024];fn random_transform_data(rng: &mut SmallRng,@@ -75,7 +75,7 @@ fn run<B: Backend>(#[rstest]#[test_attr(uzu_test)]-fn transform_matches_cpu(#[values(5120, 6144, 17408)] columns: u32) {+fn transform_matches_cpu(#[values(4096, 5120, 6144, 6656, 17408, 19968)] columns: u32) {let mut rng = SmallRng::seed_from_u64(u64::from(columns));for batch in BATCHES {let (rht_factors, mixing, mut input) = random_transform_data(&mut rng, columns, batch);