Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Radix top-k: single-dispatch exact select (#1102)

CCYehf271cdfc3e0f
diff --git a/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rs
index ed1369857..90e204a30 100644
--- a/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rs
@@ -6,10 +6,7 @@ pub const MAX_K: u32 = 512;
pub trait RadixTopKSmall: Sized + Send + Sync {
type Backend: Backend<Kernels: Kernels<RadixTopKSmall = Self>>;
- fn new(
- context: &<Self::Backend as Backend>::Context,
- columns: u32,
- ) -> Result<Self, <Self::Backend as Backend>::Error>;
+ fn new(context: &<Self::Backend as Backend>::Context) -> Result<Self, <Self::Backend as Backend>::Error>;
fn encode(
&self,
@@ -17,7 +14,9 @@ pub trait RadixTopKSmall: Sized + Send + Sync {
output_ids: impl BufferMut<Backend = Self::Backend>,
output_scores: impl BufferMut<Backend = Self::Backend>,
rows: u32,
+ columns: u32,
k: u32,
+ sorted: bool,
command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), <Self::Backend as Backend>::Error>;
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs
index c8e226395..031ff7d93 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs
@@ -13,21 +13,13 @@ use crate::{
utils::pointers::{SendPtr, SendPtrMut},
};
-pub struct CpuRadixTopKSmall {
- columns: usize,
-}
+pub struct CpuRadixTopKSmall;
impl RadixTopKSmall for CpuRadixTopKSmall {
type Backend = Cpu;
- fn new(
- _context: &CpuContext,
- columns: u32,
- ) -> Result<Self, CpuError> {
- assert!(columns > 0);
- Ok(Self {
- columns: columns as usize,
- })
+ fn new(_context: &CpuContext) -> Result<Self, CpuError> {
+ Ok(Self)
}
fn encode(
@@ -36,11 +28,13 @@ impl RadixTopKSmall for CpuRadixTopKSmall {
output_ids: impl BufferMut<Backend = Cpu>,
output_scores: impl BufferMut<Backend = Cpu>,
rows: u32,
+ columns: u32,
k: u32,
+ sorted: bool,
command_buffer: &mut CpuCommandBufferEncoding,
) -> Result<(), CpuError> {
let rows = rows as usize;
- let columns = self.columns;
+ let columns = columns as usize;
let k = k as usize;
assert!(rows > 0 && k > 0 && k <= MAX_K as usize && k <= columns);
let (input, input_range) = input.parts();
@@ -68,7 +62,11 @@ impl RadixTopKSmall for CpuRadixTopKSmall {
indices.select_nth_unstable_by(k, compare);
indices.truncate(k);
}
- indices.sort_unstable_by(compare);
+ if sorted {
+ indices.sort_unstable_by(compare);
+ } else {
+ indices.sort_unstable();
+ }
for (rank, index) in indices.into_iter().enumerate() {
output_ids[row * k + rank] = index as u32;
output_scores[row * k + rank] = values[index];
diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/top_k.h b/crates/uzu-engine/src/backends/metal/kernel/common/top_k.h
index c17434d58..5be10173b 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/common/top_k.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/common/top_k.h
@@ -15,8 +15,4 @@ static inline float top_k_score_from_key(uint key) {
return as_type<float>(bits);
}
-// Larger keys follow the f32 total order; equal scores prefer smaller indices.
-static inline ulong top_k_ordered_key(float score, uint index, uint index_bits) {
- const ulong index_mask = (1ul << index_bits) - 1ul;
- return (ulong(top_k_score_key(score)) << index_bits) | (index_mask - ulong(index));
-}
+static inline uint top_k_bf16_key(ushort bits) { return uint(bits ^ ((bits & 0x8000) ? 0xffff : 0x8000)); }
diff --git a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal
index 3bc7a2e97..c6826adb0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal
@@ -2,218 +2,189 @@
#include <metal_atomic>
#include "common/defines.h"
#include "common/dsl.h"
+#include "common/thread_context.h"
#include "common/top_k.h"
using namespace metal;
-#define THREADS_PER_TG 256
+#define THREADS_PER_TG 512
+#define SIMDGROUPS_PER_TG (THREADS_PER_TG / METAL_SIMD_SIZE)
+
constant uint MAX_K = 512;
-constant uint RADIX_BITS = 10;
+constant uint RADIX_BITS = 8;
constant uint BUCKETS = 1 << RADIX_BITS;
-constant uint BUCKETS_PER_THREAD = BUCKETS / THREADS_PER_TG;
-constant uint VECTOR_WIDTH = 4;
-
-METAL_FUNC bool arrive_last(device atomic_uint* count, uint expected) {
- atomic_thread_fence(mem_flags::mem_device, memory_order_seq_cst, thread_scope_device);
- const bool last = atomic_fetch_add_explicit(count, 1u, memory_order_relaxed) == expected - 1;
- if (last)
- atomic_thread_fence(mem_flags::mem_device, memory_order_seq_cst, thread_scope_device);
- return last;
-}
-
-METAL_FUNC void reset_arrival(device atomic_uint* count) {
- atomic_thread_fence(mem_flags::mem_device, memory_order_seq_cst, thread_scope_device);
- atomic_store_explicit(count, 0u, memory_order_relaxed);
-}
-
-template <typename Visitor>
-METAL_FUNC void visit_partition_keys(
- const device bfloat* input,
+constant uint HISTOGRAM_REPLICAS = 8;
+constant uint HISTOGRAM_SIZE = BUCKETS * HISTOGRAM_REPLICAS;
+constant uint VECTOR_WIDTH = 8;
+constant uint VECTORS_PER_THREAD = 2;
+constant uint SCORES_PER_THREAD = VECTOR_WIDTH * VECTORS_PER_THREAD;
+constant uint SCORES_PER_BLOCK = SCORES_PER_THREAD * THREADS_PER_TG;
+constant uint HALF_BITS = 16;
+constant uint HALF_MASK = (1 << HALF_BITS) - 1;
+
+METAL_FUNC void load_row_keys(
+ const device ushort* row_scores,
uint columns,
- uint index_bits,
- uint partition,
- uint partitions,
- uint lid,
- Visitor visit
+ uint first_column,
+ thread uint (&keys)[SCORES_PER_THREAD]
) {
- if (columns % VECTOR_WIDTH != 0) {
- const uint begin = columns * partition / partitions;
- const uint end = columns * (partition + 1u) / partitions;
- for (uint column = begin + lid; column < end; column += THREADS_PER_TG)
- visit(top_k_ordered_key(float(input[column]), column, index_bits));
- return;
+ const bool vector_aligned = (reinterpret_cast<ulong>(row_scores) & 15) == 0;
+ METAL_PRAGMA_UNROLL
+ for (uint vector = 0; vector < VECTORS_PER_THREAD; ++vector) {
+ const uint vector_column = first_column + vector * VECTOR_WIDTH;
+ if (vector_aligned && vector_column + VECTOR_WIDTH <= columns) {
+ const uint4 packed_scores = *reinterpret_cast<const device uint4*>(row_scores + vector_column);
+ METAL_PRAGMA_UNROLL
+ for (uint lane = 0; lane < VECTOR_WIDTH; ++lane) {
+ keys[vector * VECTOR_WIDTH + lane] = top_k_bf16_key(ushort(packed_scores[lane / 2] >> (16 * (lane % 2))));
+ }
+ } else {
+ for (uint lane = 0; lane < VECTOR_WIDTH; ++lane) {
+ const uint column = vector_column + lane;
+ keys[vector * VECTOR_WIDTH + lane] = column < columns ? top_k_bf16_key(row_scores[column]) : 0;
+ }
+ }
}
- const device vec<bfloat, VECTOR_WIDTH>* input4 = reinterpret_cast<const device vec<bfloat, VECTOR_WIDTH>*>(input);
- const uint vector_columns = columns / VECTOR_WIDTH;
- const uint vector_begin = vector_columns * partition / partitions;
- const uint vector_end = vector_columns * (partition + 1u) / partitions;
- for (uint vector = vector_begin + lid; vector < vector_end; vector += THREADS_PER_TG) {
- const vec<bfloat, VECTOR_WIDTH> values = input4[vector];
- for (uint lane = 0; lane < VECTOR_WIDTH; ++lane)
- visit(top_k_ordered_key(float(values[lane]), vector * VECTOR_WIDTH + lane, index_bits));
+}
+
+METAL_FUNC uint2
+threadgroup_prefix_sum(uint value, threadgroup uint* simdgroup_totals, const ThreadContext thread_context) {
+ const uint simd_prefix = simd_prefix_exclusive_sum(value);
+ if (thread_context.simd_lane_id == METAL_SIMD_SIZE - 1) {
+ simdgroup_totals[thread_context.simdgroup_index] = simd_prefix + value;
}
- if (partition + 1 == partitions) {
- for (uint column = vector_columns * VECTOR_WIDTH + lid; column < columns; column += THREADS_PER_TG)
- visit(top_k_ordered_key(float(input[column]), column, index_bits));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ uint previous_simdgroups_total = 0;
+ uint total = 0;
+ for (uint simdgroup = 0; simdgroup < SIMDGROUPS_PER_TG; ++simdgroup) {
+ const uint simdgroup_total = simdgroup_totals[simdgroup];
+ previous_simdgroups_total += simdgroup < thread_context.simdgroup_index ? simdgroup_total : 0;
+ total += simdgroup_total;
}
+ return uint2(simd_prefix + previous_simdgroups_total, total);
}
-KERNEL(RadixTopKSmallPass)(
+KERNEL(RadixTopKSmall)(
const device bfloat* input,
- device atomic_uint* partial_histograms,
- device ulong* prefixes,
- device ulong* prefix_masks,
- device uint* ranks,
- device atomic_uint* done_counts,
+ device uint* output_ids,
+ device float* output_scores,
constant uint& rows,
+ constant uint& columns,
constant uint& k,
- constant uint& pass,
- const uint columns SPECIALIZE,
- const uint partitions SPECIALIZE,
- threadgroup uint histogram[BUCKETS],
- threadgroup uint simd_prefixes[THREADS_PER_TG / METAL_SIMD_SIZE],
- threadgroup uint& is_last,
- const uint group GROUPS(rows* partitions),
+ const bool sorted SPECIALIZE,
+ threadgroup atomic_uint histograms[2 * HISTOGRAM_SIZE],
+ threadgroup uint simdgroup_totals[2 * SIMDGROUPS_PER_TG],
+ threadgroup uint& chosen_bucket,
+ threadgroup uint& chosen_rank,
+ threadgroup uint winner_orders[MAX_K],
+ threadgroup uint winner_columns[MAX_K],
+ const ThreadContext thread_context,
+ const uint row GROUPS(rows),
const uint lid THREADS(THREADS_PER_TG)
) {
- const uint row = group / partitions;
- const uint partition = group % partitions;
- const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);
- const uint passes = (32u + index_bits + RADIX_BITS - 1u) / RADIX_BITS;
- const uint shift = (passes - pass - 1u) * RADIX_BITS;
const device bfloat* row_input = input + ulong(row) * columns;
- const ulong prefix = prefixes[row];
- const ulong mask = prefix_masks[row];
-
- // Build this partition's histogram.
- device atomic_uint* partition_histogram = partial_histograms + (row * partitions + partition) * BUCKETS;
- for (uint bucket = lid; bucket < BUCKETS; bucket += THREADS_PER_TG)
- atomic_store_explicit(&partition_histogram[bucket], 0u, memory_order_relaxed);
- // Order this partition's clears before its histogram updates.
- threadgroup_barrier(mem_flags::mem_device);
- visit_partition_keys(row_input, columns, index_bits, partition, partitions, lid, [&](ulong key) {
- if ((key & mask) == prefix)
- atomic_fetch_add_explicit(&partition_histogram[(key >> shift) & (BUCKETS - 1)], 1u, memory_order_relaxed);
- });
- threadgroup_barrier(mem_flags::mem_device);
-
- // The last partition merges the row.
- if (lid == 0)
- is_last = arrive_last(&done_counts[row], partitions);
- threadgroup_barrier(mem_flags::mem_threadgroup | mem_flags::mem_device);
- if (!is_last)
- return;
+ const device ushort* row_scores = reinterpret_cast<const device ushort*>(row_input);
+ device uint* row_ids = output_ids + row * k;
+ device float* row_output_scores = output_scores + row * k;
- for (uint bucket = lid; bucket < BUCKETS; bucket += THREADS_PER_TG) {
- uint count = 0;
- for (uint p = 0; p < partitions; ++p)
- count +=
- atomic_load_explicit(&partial_histograms[(row * partitions + p) * BUCKETS + bucket], memory_order_relaxed);
- histogram[bucket] = count;
+ for (uint index = lid; index < 2 * HISTOGRAM_SIZE; index += THREADS_PER_TG) {
+ atomic_store_explicit(&histograms[index], 0u, memory_order_relaxed);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- uint sum = 0;
- const uint begin = BUCKETS - (lid + 1) * BUCKETS_PER_THREAD;
- for (uint offset = 0; offset < BUCKETS_PER_THREAD; ++offset)
- sum += histogram[begin + offset];
- uint higher = simd_prefix_exclusive_sum(sum);
- if ((lid + 1) % METAL_SIMD_SIZE == 0)
- simd_prefixes[lid / METAL_SIMD_SIZE] = higher + sum;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (lid < METAL_SIMD_SIZE) {
- const uint simdgroups = THREADS_PER_TG / METAL_SIMD_SIZE;
- const uint simd_sum = lid < simdgroups ? simd_prefixes[lid] : 0;
- const uint simd_prefix = simd_prefix_exclusive_sum(simd_sum);
- if (lid < simdgroups)
- simd_prefixes[lid] = simd_prefix;
- }
-
- uint rank = pass == 0 ? k - 1 : ranks[row];
-
- threadgroup_barrier(mem_flags::mem_threadgroup | mem_flags::mem_device);
- higher += simd_prefixes[lid / METAL_SIMD_SIZE];
+ uint threshold_key = 0;
+ uint rank = k - 1;
+ METAL_PRAGMA_UNROLL
+ for (uint pass = 0; pass < 2; ++pass) {
+ const uint shift = RADIX_BITS * (1 - pass);
+ threadgroup atomic_uint* histogram = histograms + pass * HISTOGRAM_SIZE;
+ for (uint first_column = lid * SCORES_PER_THREAD; first_column < columns; first_column += SCORES_PER_BLOCK) {
+ uint keys[SCORES_PER_THREAD];
+ load_row_keys(row_scores, columns, first_column, keys);
+ METAL_PRAGMA_UNROLL
+ for (uint index = 0; index < SCORES_PER_THREAD; ++index) {
+ if ((keys[index] >> (shift + RADIX_BITS)) == threshold_key) {
+ const uint replica = thread_context.simd_lane_id % HISTOGRAM_REPLICAS;
+ const uint bucket = (keys[index] >> shift) & (BUCKETS - 1);
+ atomic_fetch_add_explicit(&histogram[bucket * HISTOGRAM_REPLICAS + replica], 1u, memory_order_relaxed);
+ }
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- if (higher <= rank && rank < higher + sum) {
- rank -= higher;
- const uint group_begin = BUCKETS - (lid + 1) * BUCKETS_PER_THREAD;
- for (uint offset = 0; offset < BUCKETS_PER_THREAD; ++offset) {
- const uint bucket = group_begin + BUCKETS_PER_THREAD - offset - 1;
- const uint count = histogram[bucket];
- if (rank < count) {
- prefixes[row] |= ulong(bucket) << shift;
- prefix_masks[row] |= ulong(BUCKETS - 1) << shift;
- break;
+ const uint bucket = BUCKETS - 1 - lid;
+ uint count = 0;
+ if (lid < BUCKETS) {
+ for (uint replica = 0; replica < HISTOGRAM_REPLICAS; ++replica) {
+ count += atomic_load_explicit(&histogram[bucket * HISTOGRAM_REPLICAS + replica], memory_order_relaxed);
}
- rank -= count;
}
- ranks[row] = rank;
+ const uint larger_keys = threadgroup_prefix_sum(count, simdgroup_totals, thread_context).x;
+ if (larger_keys <= rank && rank < larger_keys + count) {
+ chosen_bucket = bucket;
+ chosen_rank = rank - larger_keys;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ threshold_key = threshold_key << RADIX_BITS | chosen_bucket;
+ rank = chosen_rank;
}
- threadgroup_barrier(mem_flags::mem_device);
- if (lid == 0)
- reset_arrival(&done_counts[row]);
-}
-KERNEL(RadixTopKSmallCollect)(
- const device bfloat* input,
- device uint* output_ids,
- device float* output_scores,
- const device ulong* prefixes,
- device atomic_uint* selected_counts,
- device atomic_uint* done_counts,
- device coherent(device) ulong* selected_keys,
- constant uint& rows,
- constant uint& k,
- const uint columns SPECIALIZE,
- const uint partitions SPECIALIZE,
- threadgroup ulong sorted_keys[MAX_K],
- threadgroup uint& is_last,
- const uint group GROUPS(rows* partitions),
- const uint lid THREADS(THREADS_PER_TG)
-) {
- const uint row = group / partitions;
- const uint partition = group % partitions;
- const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);
- const device bfloat* row_input = input + ulong(row) * columns;
- const ulong prefix = prefixes[row];
-
- visit_partition_keys(row_input, columns, index_bits, partition, partitions, lid, [&](ulong key) {
- if (key >= prefix) {
- const uint index = atomic_fetch_add_explicit(&selected_counts[row], 1u, memory_order_relaxed);
- // The final radix prefix selects exactly k unique keys.
- selected_keys[row * k + index] = key;
+ const uint tie_winner_count = rank + 1;
+ const uint above_threshold_count = k - tie_winner_count;
+ uint above_threshold_before = 0;
+ uint ties_before = 0;
+ for (uint block = 0; block * SCORES_PER_BLOCK < columns; ++block) {
+ const uint first_column = block * SCORES_PER_BLOCK + lid * SCORES_PER_THREAD;
+ uint keys[SCORES_PER_THREAD];
+ load_row_keys(row_scores, columns, first_column, keys);
+ uint thread_above_threshold = 0;
+ uint thread_ties = 0;
+ METAL_PRAGMA_UNROLL
+ for (uint index = 0; index < SCORES_PER_THREAD; ++index) {
+ thread_above_threshold += keys[index] > threshold_key;
+ thread_ties += keys[index] == threshold_key && first_column + index < columns;
}
- });
- threadgroup_barrier(mem_flags::mem_device);
- if (lid == 0)
- is_last = arrive_last(&done_counts[row], partitions);
- threadgroup_barrier(mem_flags::mem_threadgroup | mem_flags::mem_device);
- if (!is_last)
- return;
-
- for (uint index = lid; index < MAX_K; index += THREADS_PER_TG)
- sorted_keys[index] = index < k ? selected_keys[row * k + index] : 0;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint size = 2; size <= MAX_K; size <<= 1) {
- for (uint stride = size >> 1; stride; stride >>= 1) {
- for (uint index = lid; index < MAX_K; index += THREADS_PER_TG) {
- const uint other = index ^ stride;
- if (other > index) {
- const bool descending = (index & size) == 0;
- if (descending ? sorted_keys[index] < sorted_keys[other] : sorted_keys[index] > sorted_keys[other]) {
- const ulong key = sorted_keys[index];
- sorted_keys[index] = sorted_keys[other];
- sorted_keys[other] = key;
- }
+ const uint2 prefix_and_total = threadgroup_prefix_sum(
+ thread_above_threshold | thread_ties << HALF_BITS,
+ simdgroup_totals + (block % 2) * SIMDGROUPS_PER_TG,
+ thread_context
+ );
+ uint above_threshold_slot = above_threshold_before + (prefix_and_total.x & HALF_MASK);
+ uint tie_rank = ties_before + (prefix_and_total.x >> HALF_BITS);
+ METAL_PRAGMA_UNROLL
+ for (uint index = 0; index < SCORES_PER_THREAD; ++index) {
+ const uint column = first_column + index;
+ const bool above_threshold = keys[index] > threshold_key;
+ const bool tie = keys[index] == threshold_key && column < columns;
+ const uint slot = above_threshold_slot + min(tie_rank, tie_winner_count);
+ if (above_threshold || (tie && tie_rank < tie_winner_count)) {
+ if (sorted) {
+ winner_orders[slot] = keys[index] << HALF_BITS | (HALF_MASK - slot);
+ winner_columns[slot] = column;
+ } else {
+ row_ids[slot] = column;
+ row_output_scores[slot] = float(row_input[column]);
}
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ above_threshold_slot += above_threshold;
+ tie_rank += tie;
+ }
+ above_threshold_before += prefix_and_total.y & HALF_MASK;
+ ties_before += prefix_and_total.y >> HALF_BITS;
+ if (above_threshold_before == above_threshold_count && ties_before >= tie_winner_count) {
+ break;
}
}
- for (uint index = lid; index < k; index += THREADS_PER_TG) {
- const ulong key = sorted_keys[index];
- const ulong mask = (1ul << index_bits) - 1;
- output_ids[row * k + index] = uint(mask - (key & mask));
- output_scores[row * k + index] = top_k_score_from_key(uint(key >> index_bits));
+
+ if (sorted) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint slot = lid; slot < k; slot += THREADS_PER_TG) {
+ uint winners_ahead = 0;
+ for (uint other_slot = 0; other_slot < k; ++other_slot) {
+ winners_ahead += winner_orders[other_slot] > winner_orders[slot];
+ }
+ row_ids[winners_ahead] = winner_columns[slot];
+ row_output_scores[winners_ahead] = float(row_input[winner_columns[slot]]);
+ }
}
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rs
index a49ec1410..2f16c38c6 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rs
@@ -1,55 +1,26 @@
-use std::mem::size_of;
+use std::sync::OnceLock;
-use super::{RadixTopKSmallCollectMetalKernel, RadixTopKSmallPassMetalKernel};
+use super::RadixTopKSmallMetalKernel;
use crate::backends::{
common::{
- BufferMut, BufferRef, CommandBufferEncoding,
+ BufferMut, BufferRef,
kernel::radix_top_k_small::{MAX_K, RadixTopKSmall},
},
metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},
};
-const DEFAULT_PARTITIONS: u32 = 4;
-const LARGE_COLUMNS_MIN: u32 = 131_072;
-const RADIX_BITS: u32 = 10;
-const RADIX_BUCKETS: usize = 1 << RADIX_BITS;
-
-const fn partitions(columns: u32) -> u32 {
- if columns >= LARGE_COLUMNS_MIN {
- 8
- } else {
- DEFAULT_PARTITIONS
- }
-}
-
pub struct MetalRadixTopKSmall {
- pass: RadixTopKSmallPassMetalKernel,
- collect: RadixTopKSmallCollectMetalKernel,
- columns: u32,
- partitions: u32,
- passes: u32,
+ kernels: [OnceLock<RadixTopKSmallMetalKernel>; 2],
}
impl RadixTopKSmall for MetalRadixTopKSmall {
type Backend = Metal;
- fn new(
- context: &MetalContext,
- columns: u32,
- ) -> Result<Self, MetalError> {
- assert!(columns > 0);
- let index_bits = if columns <= 1 {
- 1
- } else {
- u32::BITS - (columns - 1).leading_zeros()
- };
- let partitions = partitions(columns);
+ fn new(context: &MetalContext) -> Result<Self, MetalError> {
+ let kernels = [OnceLock::new(), OnceLock::new()];
+ kernels[1].set(RadixTopKSmallMetalKernel::new(context, true)?).ok();
Ok(Self {
- pass: RadixTopKSmallPassMetalKernel::new(context, columns, partitions)?,
- collect: RadixTopKSmallCollectMetalKernel::new(context, columns, partitions)?,
- columns,
- partitions,
- passes: (u32::BITS + index_bits).div_ceil(RADIX_BITS),
+ kernels,
})
}
@@ -59,50 +30,17 @@ impl RadixTopKSmall for MetalRadixTopKSmall {
output_ids: impl BufferMut<Backend = Metal>,
output_scores: impl BufferMut<Backend = Metal>,
rows: u32,
+ columns: u32,
k: u32,
+ sorted: bool,
command_buffer: &mut MetalCommandBufferEncoding,
) -> Result<(), MetalError> {
- assert!(rows > 0 && k > 0 && k <= MAX_K && k <= self.columns);
- let rows = rows as usize;
- let mut histograms =
- command_buffer.allocate_scratch(rows * self.partitions as usize * RADIX_BUCKETS * size_of::<u32>())?;
- let mut prefixes = command_buffer.allocate_scratch(rows * size_of::<u64>())?;
- let mut masks = command_buffer.allocate_scratch(rows * size_of::<u64>())?;
- let mut ranks = command_buffer.allocate_scratch(rows * size_of::<u32>())?;
- let mut counts = command_buffer.allocate_scratch(rows * size_of::<u32>())?;
- let mut done = command_buffer.allocate_scratch(rows * size_of::<u32>())?;
- let mut keys = command_buffer.allocate_scratch(rows * k as usize * size_of::<u64>())?;
- command_buffer.encode_fill(&mut prefixes, 0);
- command_buffer.encode_fill(&mut masks, 0);
- command_buffer.encode_fill(&mut counts, 0);
- command_buffer.encode_fill(&mut done, 0);
-
- for pass in 0..self.passes {
- self.pass.encode(
- input,
- &mut histograms,
- &mut prefixes,
- &mut masks,
- &mut ranks,
- &mut done,
- rows as u32,
- k,
- pass,
- command_buffer,
- );
- }
- self.collect.encode(
- input,
- output_ids,
- output_scores,
- &prefixes,
- &mut counts,
- &mut done,
- &mut keys,
- rows as u32,
- k,
- command_buffer,
- );
+ assert!(rows > 0 && columns > 0 && k > 0 && k <= MAX_K && k <= columns);
+ let context = command_buffer.context.clone();
+ let kernel = self.kernels[usize::from(sorted)].get_or_init(|| {
+ RadixTopKSmallMetalKernel::new(&context, sorted).expect("Failed to create radix top-k kernel")
+ });
+ kernel.encode(input, output_ids, output_scores, rows, columns, k, command_buffer);
Ok(())
}
}
diff --git a/crates/uzu-engine/src/backends/webgpu/kernel.rs b/crates/uzu-engine/src/backends/webgpu/kernel.rs
index c7c4a2129..11a5382f2 100644
--- a/crates/uzu-engine/src/backends/webgpu/kernel.rs
+++ b/crates/uzu-engine/src/backends/webgpu/kernel.rs
@@ -92,10 +92,7 @@ impl MatmulKernel for Unsupported<WebGPU> {
impl RadixTopKSmall for Unsupported<WebGPU> {
type Backend = WebGPU;
- fn new(
- _context: &WebGPUContext,
- _columns: u32,
- ) -> Result<Self, WebGPUError> {
+ fn new(_context: &WebGPUContext) -> Result<Self, WebGPUError> {
Err(WebGPUError::NotSupported)
}
@@ -105,7 +102,9 @@ impl RadixTopKSmall for Unsupported<WebGPU> {
_output_ids: impl BufferMut<Backend = WebGPU>,
_output_scores: impl BufferMut<Backend = WebGPU>,
_rows: u32,
+ _columns: u32,
_k: u32,
+ _sorted: bool,
_command_buffer: &mut WebGPUCommandBufferEncoding,
) -> Result<(), WebGPUError> {
Err(WebGPUError::NotSupported)
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index 8499e59e5..714f95595 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -197,7 +197,6 @@ pub enum WeaverEncodeError<B: Backend> {
impl<B: Backend> Weaver<B> {
pub async fn new(
- vocab_size: u32,
config: &WeaverConfig,
readout_input_hadamard_factors: Option<Arc<B::GlobalGpuBuffer>>,
parameter_tree: &ParameterTree<'_, B>,
@@ -321,8 +320,7 @@ impl<B: Backend> Weaver<B> {
})
.transpose()
.map_err(WeaverNewError::Backend)?;
- let top_k =
- <B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(WeaverNewError::Backend)?;
+ let top_k = <B::Kernels as Kernels>::RadixTopKSmall::new(context).map_err(WeaverNewError::Backend)?;
let top_children = <B::Kernels as Kernels>::WeaverSelectKernel::new(context, 256, Children, false)
.map_err(WeaverNewError::Backend)?;
let top_children_with_prune_noise =
@@ -687,7 +685,9 @@ impl<B: Backend> Weaver<B> {
&mut candidate_ids,
&mut candidate_logits,
pool_depth_count,
+ vocab_size,
self.candidate_pool_size,
+ true,
command_buffer,
)
.map_err(WeaverEncodeError::Backend)?;
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index fccbe2d7c..812ebe301 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -132,7 +132,6 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
let weaver = if let Some(weaver_config) = &config.weaver_config {
Some(
Weaver::new(
- config.draft_config.vocab_size,
weaver_config,
target_embedding.readout_input_hadamard_factors(),
&speculator_tree.subtree("weaver"),
diff --git a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs
index ee9a5d2ca..10708a306 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs
@@ -33,14 +33,17 @@ fn radix_top_k_small<B: Backend>(
rows: usize,
columns: usize,
k: usize,
+ sorted: bool,
) -> (Vec<u32>, Vec<f32>) {
let context = create_context::<B>();
let input = create_buffer_with_data::<B, bf16>(&context, input);
let mut ids = create_buffer::<B, u32>(&context, rows * k);
let mut scores = create_buffer::<B, f32>(&context, rows * k);
- let kernel = <B::Kernels as Kernels>::RadixTopKSmall::new(&context, columns as u32).unwrap();
+ let kernel = <B::Kernels as Kernels>::RadixTopKSmall::new(&context).unwrap();
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
- kernel.encode(&input, &mut ids, &mut scores, rows as u32, k as u32, &mut command_buffer).unwrap();
+ kernel
+ .encode(&input, &mut ids, &mut scores, rows as u32, columns as u32, k as u32, sorted, &mut command_buffer)
+ .unwrap();
block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
(buffer_to_vec(&*context, &ids), buffer_to_vec(&*context, &scores))
}
@@ -50,13 +53,18 @@ fn reference(
rows: usize,
columns: usize,
k: usize,
+ sorted: bool,
) -> (Vec<u32>, Vec<f32>) {
let mut ids = Vec::with_capacity(rows * k);
let mut scores = Vec::with_capacity(rows * k);
for values in input.chunks_exact(columns) {
let mut row = (0..columns).collect::<Vec<_>>();
row.sort_unstable_by(|&left, &right| values[right].total_cmp(&values[left]).then_with(|| left.cmp(&right)));
- for index in row.into_iter().take(k) {
+ row.truncate(k);
+ if !sorted {
+ row.sort_unstable();
+ }
+ for index in row {
ids.push(index as u32);
scores.push(values[index]);
}
@@ -84,6 +92,7 @@ fn radix_top_k_small_matches_cpu() {
(3, 1025, 1),
(3, 1025, TARGET_K - 1),
(3, 1025, TARGET_K),
+ (1, 262_145, TARGET_K),
(15, TARGET_COLUMNS, TARGET_K),
] {
let mut input = values(rows, columns).into_iter().map(bf16::from_f32).collect::<Vec<_>>();
@@ -99,17 +108,20 @@ fn radix_top_k_small_matches_cpu() {
];
let special_count = special.len().min(input.len());
input[..special_count].copy_from_slice(&special[..special_count]);
- let expected = reference(&input.iter().map(|value| value.to_f32()).collect::<Vec<_>>(), rows, columns, k);
- assert_output(&radix_top_k_small::<Cpu>(&input, rows, columns, k), &expected, shape);
- for_each_non_cpu_backend!(|B| {
- let actual = radix_top_k_small::<B>(&input, rows, columns, k);
- assert_output(&actual, &expected, shape);
- if columns == TARGET_COLUMNS {
- for _ in 0..2 {
- assert_output(&radix_top_k_small::<B>(&input, rows, columns, k), &expected, shape);
+ for sorted in [true, false] {
+ let expected =
+ reference(&input.iter().map(|value| value.to_f32()).collect::<Vec<_>>(), rows, columns, k, sorted);
+ assert_output(&radix_top_k_small::<Cpu>(&input, rows, columns, k, sorted), &expected, shape);
+ for_each_non_cpu_backend!(|B| {
+ let actual = radix_top_k_small::<B>(&input, rows, columns, k, sorted);
+ assert_output(&actual, &expected, shape);
+ if columns == TARGET_COLUMNS {
+ for _ in 0..2 {
+ assert_output(&radix_top_k_small::<B>(&input, rows, columns, k, sorted), &expected, shape);
+ }
}
- }
- });
+ });
+ }
}
}
@@ -128,13 +140,23 @@ fn benchmark_radix_top_k_small() {
);
let mut ids = create_buffer::<Metal, u32>(&context, ROWS * TARGET_K);
let mut scores = create_buffer::<Metal, f32>(&context, ROWS * TARGET_K);
- let kernel =
- <<Metal as Backend>::Kernels as Kernels>::RadixTopKSmall::new(&context, TARGET_COLUMNS as u32).unwrap();
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RadixTopKSmall::new(&context).unwrap();
let mut run = || {
let start = Instant::now();
let mut command_buffer = context.create_command_buffer(None, None).unwrap();
for _ in 0..BATCH {
- kernel.encode(&input, &mut ids, &mut scores, ROWS as u32, TARGET_K as u32, &mut command_buffer).unwrap();
+ kernel
+ .encode(
+ &input,
+ &mut ids,
+ &mut scores,
+ ROWS as u32,
+ TARGET_COLUMNS as u32,
+ TARGET_K as u32,
+ true,
+ &mut command_buffer,
+ )
+ .unwrap();
}
let completed = block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();
(completed.gpu_execution_time().unwrap().div_f64(BATCH as f64), start.elapsed().div_f64(BATCH as f64))