Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

trellis transform for muse glimmer (#902)

CCYeh615c09605f8d
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs
index 3455d9099..d8e9685f0 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/activation_transform/trellis.rs
@@ -32,7 +32,7 @@ fn rotate_token(
}
#[kernel(TrellisTransform)]
-#[variants(DIMENSION, 5120, 6144, 17408)]
+#[variants(DIMENSION, 4096, 5120, 6144, 6656, 17408, 19968)]
pub fn trellis_transform<const DIMENSION: u32>(
input: *const bf16,
rht_factors: *const f32,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal
index 0ea879905..a365e1f68 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/activation_transform/trellis.metal
@@ -12,7 +12,7 @@ namespace trellis = uzu::trellis;
UZU_CONST uint SIMDGROUPS_PER_THREADGROUP = THREADS_PER_THREADGROUP / METAL_SIMD_SIZE;
-UZU_CONST uint MAX_MIXING_DIMENSION = 17;
+UZU_CONST uint THREADGROUP_MEMORY_LIMIT = 32768;
static METAL_FUNC float butterfly(float value, ushort lane, ushort lane_stride) {
const float other = simd_shuffle_xor(value, lane_stride);
@@ -27,34 +27,42 @@ static METAL_FUNC uint hadamard_index_before_transpose(ushort lane, ushort simdg
return METAL_SIMD_SIZE * (simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index) + lane;
}
-template <uint HADAMARD_SIZE>
-static METAL_FUNC uint hadamard_index_after_transpose(ushort lane, ushort simdgroup, uint value_index) {
- if (HADAMARD_SIZE == 1024) {
- return METAL_SIMD_SIZE * lane + simdgroup + SIMDGROUPS_PER_THREADGROUP * value_index;
- }
- return METAL_SIMD_SIZE * (lane + METAL_SIMD_SIZE * (value_index % 2)) + simdgroup +
- SIMDGROUPS_PER_THREADGROUP * (value_index / 2);
-}
-
template <uint DIMENSION>
struct TransformLayout {
static constant constexpr uint HADAMARD_SIZE = DIMENSION & (0u - DIMENSION);
static constant constexpr uint MIXING_DIMENSION = DIMENSION / HADAMARD_SIZE;
static constant constexpr uint VALUES_PER_THREAD = HADAMARD_SIZE / THREADS_PER_THREADGROUP;
- static constant constexpr uint COLUMNS_PER_PASS = HADAMARD_SIZE == 2048 ? 1 : (MIXING_DIMENSION > 8 ? 4 : 2);
+ static constant constexpr uint IN_THREAD_VALUES = HADAMARD_SIZE > 1024 ? HADAMARD_SIZE / 1024 : 1;
+ static constant constexpr uint SECOND_LANE_STRIDE_LIMIT =
+ HADAMARD_SIZE == THREADS_PER_THREADGROUP ? METAL_SIMD_SIZE / 4 : METAL_SIMD_SIZE / 2;
+ static constant constexpr uint COLUMNS_PER_PASS = HADAMARD_SIZE >= 2048 ? 1 : (MIXING_DIMENSION > 8 ? 4 : 2);
static constant constexpr uint COLUMN_SCRATCH_SIZE = HADAMARD_SIZE + HADAMARD_SIZE / METAL_SIMD_SIZE;
static constant constexpr uint SCRATCH_SIZE = COLUMNS_PER_PASS * COLUMN_SCRATCH_SIZE;
+ static constant constexpr uint MIXING_SIZE = MIXING_DIMENSION * MIXING_DIMENSION;
+ static constant constexpr uint THREADGROUP_BYTES =
+ sizeof(float) * (SCRATCH_SIZE + SIMDGROUPS_PER_THREADGROUP + MIXING_SIZE);
typedef float SignFlippedInput[VALUES_PER_THREAD][MIXING_DIMENSION];
typedef float RotatedColumns[MIXING_DIMENSION][VALUES_PER_THREAD];
};
-template <typename Layout, uint COLUMNS>
+template <typename Layout>
+static METAL_FUNC uint hadamard_index_after_transpose(ushort lane, ushort simdgroup, uint value_index) {
+ if constexpr (Layout::HADAMARD_SIZE == THREADS_PER_THREADGROUP) {
+ constexpr uint half_simd = METAL_SIMD_SIZE / 2;
+ return METAL_SIMD_SIZE * (lane % half_simd) + half_simd * (lane / half_simd) + simdgroup;
+ } else {
+ return METAL_SIMD_SIZE * (lane + METAL_SIMD_SIZE * (value_index % Layout::IN_THREAD_VALUES)) + simdgroup +
+ SIMDGROUPS_PER_THREADGROUP * (value_index / Layout::IN_THREAD_VALUES);
+ }
+}
+
+template <typename Layout, uint COLUMNS, uint LANE_STRIDE_LIMIT = METAL_SIMD_SIZE / 2>
static METAL_FUNC void hadamard_transform_lanes(
thread float (&column_values)[COLUMNS][Layout::VALUES_PER_THREAD],
const ushort lane
) {
METAL_PRAGMA_UNROLL
- for (ushort lane_stride = 1; lane_stride <= METAL_SIMD_SIZE / 2; lane_stride <<= 1) {
+ for (ushort lane_stride = 1; lane_stride <= LANE_STRIDE_LIMIT; lane_stride <<= 1) {
METAL_PRAGMA_UNROLL
for (uint column = 0; column < COLUMNS; ++column) {
METAL_PRAGMA_UNROLL
@@ -109,21 +117,25 @@ static METAL_FUNC void rotate_columns(
for (uint column = 0; column < COLUMNS; ++column) {
METAL_PRAGMA_UNROLL
for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
- const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);
+ const uint hadamard_index = hadamard_index_after_transpose<Layout>(lane, simdgroup, value_index);
column_values[column][value_index] =
scratch[column * Layout::COLUMN_SCRATCH_SIZE + bank_conflict_free_offset(hadamard_index)];
}
}
- hadamard_transform_lanes<Layout, COLUMNS>(column_values, lane);
- if constexpr (Layout::HADAMARD_SIZE == 2048) {
+ hadamard_transform_lanes<Layout, COLUMNS, Layout::SECOND_LANE_STRIDE_LIMIT>(column_values, lane);
+ METAL_PRAGMA_UNROLL
+ for (uint value_stride = 1; value_stride < Layout::IN_THREAD_VALUES; value_stride <<= 1) {
METAL_PRAGMA_UNROLL
for (uint column = 0; column < COLUMNS; ++column) {
METAL_PRAGMA_UNROLL
- for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; value_index += 2) {
- const float lower = column_values[column][value_index];
- const float upper = column_values[column][value_index + 1];
- column_values[column][value_index] = lower + upper;
- column_values[column][value_index + 1] = lower - upper;
+ for (uint block = 0; block < Layout::VALUES_PER_THREAD; block += 2 * value_stride) {
+ METAL_PRAGMA_UNROLL
+ for (uint value_index = block; value_index < block + value_stride; ++value_index) {
+ const float lower = column_values[column][value_index];
+ const float upper = column_values[column][value_index + value_stride];
+ column_values[column][value_index] = lower + upper;
+ column_values[column][value_index + value_stride] = lower - upper;
+ }
}
}
}
@@ -141,7 +153,7 @@ static METAL_FUNC void rotate_columns(
// TODO: a columns agnostic kernel
template <uint DIMENSION>
-VARIANTS(DIMENSION, 5120, 6144, 17408)
+VARIANTS(DIMENSION, 4096, 5120, 6144, 6656, 17408, 19968)
PUBLIC KERNEL(TrellisTransform)(
device const bfloat* input,
device const float* rht_factors,
@@ -152,14 +164,17 @@ PUBLIC KERNEL(TrellisTransform)(
constant uint& batch,
threadgroup float scratch[TransformLayout<DIMENSION>::SCRATCH_SIZE],
threadgroup float simdgroup_maxima[SIMDGROUPS_PER_THREADGROUP],
- threadgroup float shared_mixing[MAX_MIXING_DIMENSION * MAX_MIXING_DIMENSION],
+ threadgroup float shared_mixing[TransformLayout<DIMENSION>::MIXING_SIZE],
const uint token GROUPS(batch),
const uint thread_index THREADS(THREADS_PER_THREADGROUP),
const ThreadContext thread_context
) {
using Layout = TransformLayout<DIMENSION>;
- static_assert(Layout::HADAMARD_SIZE == 1024 || Layout::HADAMARD_SIZE == 2048, "unsupported Hadamard size");
- static_assert(Layout::MIXING_DIMENSION <= MAX_MIXING_DIMENSION, "mixing matrix does not fit threadgroup memory");
+ static_assert(
+ Layout::HADAMARD_SIZE >= THREADS_PER_THREADGROUP && Layout::HADAMARD_SIZE <= 4096,
+ "unsupported Hadamard size"
+ );
+ static_assert(Layout::THREADGROUP_BYTES <= THREADGROUP_MEMORY_LIMIT, "does not fit threadgroup memory");
const ushort lane = ushort(thread_context.simd_lane_id);
const ushort simdgroup = ushort(thread_context.simdgroup_index);
@@ -223,7 +238,7 @@ PUBLIC KERNEL(TrellisTransform)(
for (uint mixing_index = 0; mixing_index < Layout::MIXING_DIMENSION; ++mixing_index) {
METAL_PRAGMA_UNROLL
for (uint value_index = 0; value_index < Layout::VALUES_PER_THREAD; ++value_index) {
- const uint hadamard_index = hadamard_index_after_transpose<Layout::HADAMARD_SIZE>(lane, simdgroup, value_index);
+ const uint hadamard_index = hadamard_index_after_transpose<Layout>(lane, simdgroup, value_index);
const uint column = hadamard_index * Layout::MIXING_DIMENSION + mixing_index;
const int8_t quantized = quantize_activation_int8(rotated[mixing_index][value_index], scale);
activations[token * DIMENSION + column] = quantized;
diff --git a/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs
index d952d6b02..7c08d4716 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/trellis_test.rs
@@ -19,7 +19,7 @@ const MIXING_ENTRY_BOUND: f32 = 0.6;
const INPUT_BASE_BOUND: f32 = 3.0;
const INPUT_OUTLIER_EXPONENT: i32 = 3;
const ALL_ZERO_TOKEN: usize = 1;
-const BATCHES: [u32; 4] = [1, 3, 17, 65];
+const BATCHES: [u32; 5] = [1, 8, 16, 32, 1024];
fn random_transform_data(
rng: &mut SmallRng,
@@ -75,7 +75,7 @@ fn run<B: Backend>(
#[rstest]
#[test_attr(uzu_test)]
-fn transform_matches_cpu(#[values(5120, 6144, 17408)] columns: u32) {
+fn transform_matches_cpu(#[values(4096, 5120, 6144, 6656, 17408, 19968)] columns: u32) {
let mut rng = SmallRng::seed_from_u64(u64::from(columns));
for batch in BATCHES {
let (rht_factors, mixing, mut input) = random_transform_data(&mut rng, columns, batch);