A high-performance inference engine for AI models
Radix top-k: single-dispatch exact select (#1102)
CCYehf271cdfc3e0f
diff --git a/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rsindex ed1369857..90e204a30 100644--- a/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rs+++ b/crates/uzu-engine/src/backends/common/kernel/radix_top_k_small.rs@@ -6,10 +6,7 @@ pub const MAX_K: u32 = 512;pub trait RadixTopKSmall: Sized + Send + Sync {type Backend: Backend<Kernels: Kernels<RadixTopKSmall = Self>>;- fn new(- context: &<Self::Backend as Backend>::Context,- columns: u32,- ) -> Result<Self, <Self::Backend as Backend>::Error>;+ fn new(context: &<Self::Backend as Backend>::Context) -> Result<Self, <Self::Backend as Backend>::Error>;fn encode(&self,@@ -17,7 +14,9 @@ pub trait RadixTopKSmall: Sized + Send + Sync {output_ids: impl BufferMut<Backend = Self::Backend>,output_scores: impl BufferMut<Backend = Self::Backend>,rows: u32,+ columns: u32,k: u32,+ sorted: bool,command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), <Self::Backend as Backend>::Error>;}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rsindex c8e226395..031ff7d93 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs@@ -13,21 +13,13 @@ use crate::{utils::pointers::{SendPtr, SendPtrMut},};-pub struct CpuRadixTopKSmall {- columns: usize,-}+pub struct CpuRadixTopKSmall;impl RadixTopKSmall for CpuRadixTopKSmall {type Backend = Cpu;- fn new(- _context: &CpuContext,- columns: u32,- ) -> Result<Self, CpuError> {- assert!(columns > 0);- Ok(Self {- columns: columns as usize,- })+ fn new(_context: &CpuContext) -> Result<Self, CpuError> {+ Ok(Self)}fn encode(@@ -36,11 +28,13 @@ impl RadixTopKSmall for CpuRadixTopKSmall {output_ids: impl BufferMut<Backend = Cpu>,output_scores: impl BufferMut<Backend = Cpu>,rows: u32,+ columns: u32,k: u32,+ sorted: bool,command_buffer: &mut CpuCommandBufferEncoding,) -> Result<(), CpuError> {let rows = rows as usize;- let columns = self.columns;+ let columns = columns as usize;let k = k as usize;assert!(rows > 0 && k > 0 && k <= MAX_K as usize && k <= columns);let (input, input_range) = input.parts();@@ -68,7 +62,11 @@ impl RadixTopKSmall for CpuRadixTopKSmall {indices.select_nth_unstable_by(k, compare);indices.truncate(k);}- indices.sort_unstable_by(compare);+ if sorted {+ indices.sort_unstable_by(compare);+ } else {+ indices.sort_unstable();+ }for (rank, index) in indices.into_iter().enumerate() {output_ids[row * k + rank] = index as u32;output_scores[row * k + rank] = values[index];diff --git a/crates/uzu-engine/src/backends/metal/kernel/common/top_k.h b/crates/uzu-engine/src/backends/metal/kernel/common/top_k.hindex c17434d58..5be10173b 100644--- a/crates/uzu-engine/src/backends/metal/kernel/common/top_k.h+++ b/crates/uzu-engine/src/backends/metal/kernel/common/top_k.h@@ -15,8 +15,4 @@ static inline float top_k_score_from_key(uint key) {return as_type<float>(bits);}-// Larger keys follow the f32 total order; equal scores prefer smaller indices.-static inline ulong top_k_ordered_key(float score, uint index, uint index_bits) {- const ulong index_mask = (1ul << index_bits) - 1ul;- return (ulong(top_k_score_key(score)) << index_bits) | (index_mask - ulong(index));-}+static inline uint top_k_bf16_key(ushort bits) { return uint(bits ^ ((bits & 0x8000) ? 0xffff : 0x8000)); }diff --git a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metalindex 3bc7a2e97..c6826adb0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal@@ -2,218 +2,189 @@#include <metal_atomic>#include "common/defines.h"#include "common/dsl.h"+#include "common/thread_context.h"#include "common/top_k.h"using namespace metal;-#define THREADS_PER_TG 256+#define THREADS_PER_TG 512+#define SIMDGROUPS_PER_TG (THREADS_PER_TG / METAL_SIMD_SIZE)+constant uint MAX_K = 512;-constant uint RADIX_BITS = 10;+constant uint RADIX_BITS = 8;constant uint BUCKETS = 1 << RADIX_BITS;-constant uint BUCKETS_PER_THREAD = BUCKETS / THREADS_PER_TG;-constant uint VECTOR_WIDTH = 4;--METAL_FUNC bool arrive_last(device atomic_uint* count, uint expected) {- atomic_thread_fence(mem_flags::mem_device, memory_order_seq_cst, thread_scope_device);- const bool last = atomic_fetch_add_explicit(count, 1u, memory_order_relaxed) == expected - 1;- if (last)- atomic_thread_fence(mem_flags::mem_device, memory_order_seq_cst, thread_scope_device);- return last;-}--METAL_FUNC void reset_arrival(device atomic_uint* count) {- atomic_thread_fence(mem_flags::mem_device, memory_order_seq_cst, thread_scope_device);- atomic_store_explicit(count, 0u, memory_order_relaxed);-}--template <typename Visitor>-METAL_FUNC void visit_partition_keys(- const device bfloat* input,+constant uint HISTOGRAM_REPLICAS = 8;+constant uint HISTOGRAM_SIZE = BUCKETS * HISTOGRAM_REPLICAS;+constant uint VECTOR_WIDTH = 8;+constant uint VECTORS_PER_THREAD = 2;+constant uint SCORES_PER_THREAD = VECTOR_WIDTH * VECTORS_PER_THREAD;+constant uint SCORES_PER_BLOCK = SCORES_PER_THREAD * THREADS_PER_TG;+constant uint HALF_BITS = 16;+constant uint HALF_MASK = (1 << HALF_BITS) - 1;++METAL_FUNC void load_row_keys(+ const device ushort* row_scores,uint columns,- uint index_bits,- uint partition,- uint partitions,- uint lid,- Visitor visit+ uint first_column,+ thread uint (&keys)[SCORES_PER_THREAD]) {- if (columns % VECTOR_WIDTH != 0) {- const uint begin = columns * partition / partitions;- const uint end = columns * (partition + 1u) / partitions;- for (uint column = begin + lid; column < end; column += THREADS_PER_TG)- visit(top_k_ordered_key(float(input[column]), column, index_bits));- return;+ const bool vector_aligned = (reinterpret_cast<ulong>(row_scores) & 15) == 0;+ METAL_PRAGMA_UNROLL+ for (uint vector = 0; vector < VECTORS_PER_THREAD; ++vector) {+ const uint vector_column = first_column + vector * VECTOR_WIDTH;+ if (vector_aligned && vector_column + VECTOR_WIDTH <= columns) {+ const uint4 packed_scores = *reinterpret_cast<const device uint4*>(row_scores + vector_column);+ METAL_PRAGMA_UNROLL+ for (uint lane = 0; lane < VECTOR_WIDTH; ++lane) {+ keys[vector * VECTOR_WIDTH + lane] = top_k_bf16_key(ushort(packed_scores[lane / 2] >> (16 * (lane % 2))));+ }+ } else {+ for (uint lane = 0; lane < VECTOR_WIDTH; ++lane) {+ const uint column = vector_column + lane;+ keys[vector * VECTOR_WIDTH + lane] = column < columns ? top_k_bf16_key(row_scores[column]) : 0;+ }+ }}- const device vec<bfloat, VECTOR_WIDTH>* input4 = reinterpret_cast<const device vec<bfloat, VECTOR_WIDTH>*>(input);- const uint vector_columns = columns / VECTOR_WIDTH;- const uint vector_begin = vector_columns * partition / partitions;- const uint vector_end = vector_columns * (partition + 1u) / partitions;- for (uint vector = vector_begin + lid; vector < vector_end; vector += THREADS_PER_TG) {- const vec<bfloat, VECTOR_WIDTH> values = input4[vector];- for (uint lane = 0; lane < VECTOR_WIDTH; ++lane)- visit(top_k_ordered_key(float(values[lane]), vector * VECTOR_WIDTH + lane, index_bits));+}++METAL_FUNC uint2+threadgroup_prefix_sum(uint value, threadgroup uint* simdgroup_totals, const ThreadContext thread_context) {+ const uint simd_prefix = simd_prefix_exclusive_sum(value);+ if (thread_context.simd_lane_id == METAL_SIMD_SIZE - 1) {+ simdgroup_totals[thread_context.simdgroup_index] = simd_prefix + value;}- if (partition + 1 == partitions) {- for (uint column = vector_columns * VECTOR_WIDTH + lid; column < columns; column += THREADS_PER_TG)- visit(top_k_ordered_key(float(input[column]), column, index_bits));+ threadgroup_barrier(mem_flags::mem_threadgroup);+ uint previous_simdgroups_total = 0;+ uint total = 0;+ for (uint simdgroup = 0; simdgroup < SIMDGROUPS_PER_TG; ++simdgroup) {+ const uint simdgroup_total = simdgroup_totals[simdgroup];+ previous_simdgroups_total += simdgroup < thread_context.simdgroup_index ? simdgroup_total : 0;+ total += simdgroup_total;}+ return uint2(simd_prefix + previous_simdgroups_total, total);}-KERNEL(RadixTopKSmallPass)(+KERNEL(RadixTopKSmall)(const device bfloat* input,- device atomic_uint* partial_histograms,- device ulong* prefixes,- device ulong* prefix_masks,- device uint* ranks,- device atomic_uint* done_counts,+ device uint* output_ids,+ device float* output_scores,constant uint& rows,+ constant uint& columns,constant uint& k,- constant uint& pass,- const uint columns SPECIALIZE,- const uint partitions SPECIALIZE,- threadgroup uint histogram[BUCKETS],- threadgroup uint simd_prefixes[THREADS_PER_TG / METAL_SIMD_SIZE],- threadgroup uint& is_last,- const uint group GROUPS(rows* partitions),+ const bool sorted SPECIALIZE,+ threadgroup atomic_uint histograms[2 * HISTOGRAM_SIZE],+ threadgroup uint simdgroup_totals[2 * SIMDGROUPS_PER_TG],+ threadgroup uint& chosen_bucket,+ threadgroup uint& chosen_rank,+ threadgroup uint winner_orders[MAX_K],+ threadgroup uint winner_columns[MAX_K],+ const ThreadContext thread_context,+ const uint row GROUPS(rows),const uint lid THREADS(THREADS_PER_TG)) {- const uint row = group / partitions;- const uint partition = group % partitions;- const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);- const uint passes = (32u + index_bits + RADIX_BITS - 1u) / RADIX_BITS;- const uint shift = (passes - pass - 1u) * RADIX_BITS;const device bfloat* row_input = input + ulong(row) * columns;- const ulong prefix = prefixes[row];- const ulong mask = prefix_masks[row];-- // Build this partition's histogram.- device atomic_uint* partition_histogram = partial_histograms + (row * partitions + partition) * BUCKETS;- for (uint bucket = lid; bucket < BUCKETS; bucket += THREADS_PER_TG)- atomic_store_explicit(&partition_histogram[bucket], 0u, memory_order_relaxed);- // Order this partition's clears before its histogram updates.- threadgroup_barrier(mem_flags::mem_device);- visit_partition_keys(row_input, columns, index_bits, partition, partitions, lid, [&](ulong key) {- if ((key & mask) == prefix)- atomic_fetch_add_explicit(&partition_histogram[(key >> shift) & (BUCKETS - 1)], 1u, memory_order_relaxed);- });- threadgroup_barrier(mem_flags::mem_device);-- // The last partition merges the row.- if (lid == 0)- is_last = arrive_last(&done_counts[row], partitions);- threadgroup_barrier(mem_flags::mem_threadgroup | mem_flags::mem_device);- if (!is_last)- return;+ const device ushort* row_scores = reinterpret_cast<const device ushort*>(row_input);+ device uint* row_ids = output_ids + row * k;+ device float* row_output_scores = output_scores + row * k;- for (uint bucket = lid; bucket < BUCKETS; bucket += THREADS_PER_TG) {- uint count = 0;- for (uint p = 0; p < partitions; ++p)- count +=- atomic_load_explicit(&partial_histograms[(row * partitions + p) * BUCKETS + bucket], memory_order_relaxed);- histogram[bucket] = count;+ for (uint index = lid; index < 2 * HISTOGRAM_SIZE; index += THREADS_PER_TG) {+ atomic_store_explicit(&histograms[index], 0u, memory_order_relaxed);}threadgroup_barrier(mem_flags::mem_threadgroup);- uint sum = 0;- const uint begin = BUCKETS - (lid + 1) * BUCKETS_PER_THREAD;- for (uint offset = 0; offset < BUCKETS_PER_THREAD; ++offset)- sum += histogram[begin + offset];- uint higher = simd_prefix_exclusive_sum(sum);- if ((lid + 1) % METAL_SIMD_SIZE == 0)- simd_prefixes[lid / METAL_SIMD_SIZE] = higher + sum;- threadgroup_barrier(mem_flags::mem_threadgroup);- if (lid < METAL_SIMD_SIZE) {- const uint simdgroups = THREADS_PER_TG / METAL_SIMD_SIZE;- const uint simd_sum = lid < simdgroups ? simd_prefixes[lid] : 0;- const uint simd_prefix = simd_prefix_exclusive_sum(simd_sum);- if (lid < simdgroups)- simd_prefixes[lid] = simd_prefix;- }-- uint rank = pass == 0 ? k - 1 : ranks[row];-- threadgroup_barrier(mem_flags::mem_threadgroup | mem_flags::mem_device);- higher += simd_prefixes[lid / METAL_SIMD_SIZE];+ uint threshold_key = 0;+ uint rank = k - 1;+ METAL_PRAGMA_UNROLL+ for (uint pass = 0; pass < 2; ++pass) {+ const uint shift = RADIX_BITS * (1 - pass);+ threadgroup atomic_uint* histogram = histograms + pass * HISTOGRAM_SIZE;+ for (uint first_column = lid * SCORES_PER_THREAD; first_column < columns; first_column += SCORES_PER_BLOCK) {+ uint keys[SCORES_PER_THREAD];+ load_row_keys(row_scores, columns, first_column, keys);+ METAL_PRAGMA_UNROLL+ for (uint index = 0; index < SCORES_PER_THREAD; ++index) {+ if ((keys[index] >> (shift + RADIX_BITS)) == threshold_key) {+ const uint replica = thread_context.simd_lane_id % HISTOGRAM_REPLICAS;+ const uint bucket = (keys[index] >> shift) & (BUCKETS - 1);+ atomic_fetch_add_explicit(&histogram[bucket * HISTOGRAM_REPLICAS + replica], 1u, memory_order_relaxed);+ }+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- if (higher <= rank && rank < higher + sum) {- rank -= higher;- const uint group_begin = BUCKETS - (lid + 1) * BUCKETS_PER_THREAD;- for (uint offset = 0; offset < BUCKETS_PER_THREAD; ++offset) {- const uint bucket = group_begin + BUCKETS_PER_THREAD - offset - 1;- const uint count = histogram[bucket];- if (rank < count) {- prefixes[row] |= ulong(bucket) << shift;- prefix_masks[row] |= ulong(BUCKETS - 1) << shift;- break;+ const uint bucket = BUCKETS - 1 - lid;+ uint count = 0;+ if (lid < BUCKETS) {+ for (uint replica = 0; replica < HISTOGRAM_REPLICAS; ++replica) {+ count += atomic_load_explicit(&histogram[bucket * HISTOGRAM_REPLICAS + replica], memory_order_relaxed);}- rank -= count;}- ranks[row] = rank;+ const uint larger_keys = threadgroup_prefix_sum(count, simdgroup_totals, thread_context).x;+ if (larger_keys <= rank && rank < larger_keys + count) {+ chosen_bucket = bucket;+ chosen_rank = rank - larger_keys;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ threshold_key = threshold_key << RADIX_BITS | chosen_bucket;+ rank = chosen_rank;}- threadgroup_barrier(mem_flags::mem_device);- if (lid == 0)- reset_arrival(&done_counts[row]);-}-KERNEL(RadixTopKSmallCollect)(- const device bfloat* input,- device uint* output_ids,- device float* output_scores,- const device ulong* prefixes,- device atomic_uint* selected_counts,- device atomic_uint* done_counts,- device coherent(device) ulong* selected_keys,- constant uint& rows,- constant uint& k,- const uint columns SPECIALIZE,- const uint partitions SPECIALIZE,- threadgroup ulong sorted_keys[MAX_K],- threadgroup uint& is_last,- const uint group GROUPS(rows* partitions),- const uint lid THREADS(THREADS_PER_TG)-) {- const uint row = group / partitions;- const uint partition = group % partitions;- const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);- const device bfloat* row_input = input + ulong(row) * columns;- const ulong prefix = prefixes[row];-- visit_partition_keys(row_input, columns, index_bits, partition, partitions, lid, [&](ulong key) {- if (key >= prefix) {- const uint index = atomic_fetch_add_explicit(&selected_counts[row], 1u, memory_order_relaxed);- // The final radix prefix selects exactly k unique keys.- selected_keys[row * k + index] = key;+ const uint tie_winner_count = rank + 1;+ const uint above_threshold_count = k - tie_winner_count;+ uint above_threshold_before = 0;+ uint ties_before = 0;+ for (uint block = 0; block * SCORES_PER_BLOCK < columns; ++block) {+ const uint first_column = block * SCORES_PER_BLOCK + lid * SCORES_PER_THREAD;+ uint keys[SCORES_PER_THREAD];+ load_row_keys(row_scores, columns, first_column, keys);+ uint thread_above_threshold = 0;+ uint thread_ties = 0;+ METAL_PRAGMA_UNROLL+ for (uint index = 0; index < SCORES_PER_THREAD; ++index) {+ thread_above_threshold += keys[index] > threshold_key;+ thread_ties += keys[index] == threshold_key && first_column + index < columns;}- });- threadgroup_barrier(mem_flags::mem_device);- if (lid == 0)- is_last = arrive_last(&done_counts[row], partitions);- threadgroup_barrier(mem_flags::mem_threadgroup | mem_flags::mem_device);- if (!is_last)- return;-- for (uint index = lid; index < MAX_K; index += THREADS_PER_TG)- sorted_keys[index] = index < k ? selected_keys[row * k + index] : 0;- threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint size = 2; size <= MAX_K; size <<= 1) {- for (uint stride = size >> 1; stride; stride >>= 1) {- for (uint index = lid; index < MAX_K; index += THREADS_PER_TG) {- const uint other = index ^ stride;- if (other > index) {- const bool descending = (index & size) == 0;- if (descending ? sorted_keys[index] < sorted_keys[other] : sorted_keys[index] > sorted_keys[other]) {- const ulong key = sorted_keys[index];- sorted_keys[index] = sorted_keys[other];- sorted_keys[other] = key;- }+ const uint2 prefix_and_total = threadgroup_prefix_sum(+ thread_above_threshold | thread_ties << HALF_BITS,+ simdgroup_totals + (block % 2) * SIMDGROUPS_PER_TG,+ thread_context+ );+ uint above_threshold_slot = above_threshold_before + (prefix_and_total.x & HALF_MASK);+ uint tie_rank = ties_before + (prefix_and_total.x >> HALF_BITS);+ METAL_PRAGMA_UNROLL+ for (uint index = 0; index < SCORES_PER_THREAD; ++index) {+ const uint column = first_column + index;+ const bool above_threshold = keys[index] > threshold_key;+ const bool tie = keys[index] == threshold_key && column < columns;+ const uint slot = above_threshold_slot + min(tie_rank, tie_winner_count);+ if (above_threshold || (tie && tie_rank < tie_winner_count)) {+ if (sorted) {+ winner_orders[slot] = keys[index] << HALF_BITS | (HALF_MASK - slot);+ winner_columns[slot] = column;+ } else {+ row_ids[slot] = column;+ row_output_scores[slot] = float(row_input[column]);}}- threadgroup_barrier(mem_flags::mem_threadgroup);+ above_threshold_slot += above_threshold;+ tie_rank += tie;+ }+ above_threshold_before += prefix_and_total.y & HALF_MASK;+ ties_before += prefix_and_total.y >> HALF_BITS;+ if (above_threshold_before == above_threshold_count && ties_before >= tie_winner_count) {+ break;}}- for (uint index = lid; index < k; index += THREADS_PER_TG) {- const ulong key = sorted_keys[index];- const ulong mask = (1ul << index_bits) - 1;- output_ids[row * k + index] = uint(mask - (key & mask));- output_scores[row * k + index] = top_k_score_from_key(uint(key >> index_bits));++ if (sorted) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint slot = lid; slot < k; slot += THREADS_PER_TG) {+ uint winners_ahead = 0;+ for (uint other_slot = 0; other_slot < k; ++other_slot) {+ winners_ahead += winner_orders[other_slot] > winner_orders[slot];+ }+ row_ids[winners_ahead] = winner_columns[slot];+ row_output_scores[winners_ahead] = float(row_input[winner_columns[slot]]);+ }}}diff --git a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rsindex a49ec1410..2f16c38c6 100644--- a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.rs@@ -1,55 +1,26 @@-use std::mem::size_of;+use std::sync::OnceLock;-use super::{RadixTopKSmallCollectMetalKernel, RadixTopKSmallPassMetalKernel};+use super::RadixTopKSmallMetalKernel;use crate::backends::{common::{- BufferMut, BufferRef, CommandBufferEncoding,+ BufferMut, BufferRef,kernel::radix_top_k_small::{MAX_K, RadixTopKSmall},},metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, error::MetalError},};-const DEFAULT_PARTITIONS: u32 = 4;-const LARGE_COLUMNS_MIN: u32 = 131_072;-const RADIX_BITS: u32 = 10;-const RADIX_BUCKETS: usize = 1 << RADIX_BITS;--const fn partitions(columns: u32) -> u32 {- if columns >= LARGE_COLUMNS_MIN {- 8- } else {- DEFAULT_PARTITIONS- }-}-pub struct MetalRadixTopKSmall {- pass: RadixTopKSmallPassMetalKernel,- collect: RadixTopKSmallCollectMetalKernel,- columns: u32,- partitions: u32,- passes: u32,+ kernels: [OnceLock<RadixTopKSmallMetalKernel>; 2],}impl RadixTopKSmall for MetalRadixTopKSmall {type Backend = Metal;- fn new(- context: &MetalContext,- columns: u32,- ) -> Result<Self, MetalError> {- assert!(columns > 0);- let index_bits = if columns <= 1 {- 1- } else {- u32::BITS - (columns - 1).leading_zeros()- };- let partitions = partitions(columns);+ fn new(context: &MetalContext) -> Result<Self, MetalError> {+ let kernels = [OnceLock::new(), OnceLock::new()];+ kernels[1].set(RadixTopKSmallMetalKernel::new(context, true)?).ok();Ok(Self {- pass: RadixTopKSmallPassMetalKernel::new(context, columns, partitions)?,- collect: RadixTopKSmallCollectMetalKernel::new(context, columns, partitions)?,- columns,- partitions,- passes: (u32::BITS + index_bits).div_ceil(RADIX_BITS),+ kernels,})}@@ -59,50 +30,17 @@ impl RadixTopKSmall for MetalRadixTopKSmall {output_ids: impl BufferMut<Backend = Metal>,output_scores: impl BufferMut<Backend = Metal>,rows: u32,+ columns: u32,k: u32,+ sorted: bool,command_buffer: &mut MetalCommandBufferEncoding,) -> Result<(), MetalError> {- assert!(rows > 0 && k > 0 && k <= MAX_K && k <= self.columns);- let rows = rows as usize;- let mut histograms =- command_buffer.allocate_scratch(rows * self.partitions as usize * RADIX_BUCKETS * size_of::<u32>())?;- let mut prefixes = command_buffer.allocate_scratch(rows * size_of::<u64>())?;- let mut masks = command_buffer.allocate_scratch(rows * size_of::<u64>())?;- let mut ranks = command_buffer.allocate_scratch(rows * size_of::<u32>())?;- let mut counts = command_buffer.allocate_scratch(rows * size_of::<u32>())?;- let mut done = command_buffer.allocate_scratch(rows * size_of::<u32>())?;- let mut keys = command_buffer.allocate_scratch(rows * k as usize * size_of::<u64>())?;- command_buffer.encode_fill(&mut prefixes, 0);- command_buffer.encode_fill(&mut masks, 0);- command_buffer.encode_fill(&mut counts, 0);- command_buffer.encode_fill(&mut done, 0);-- for pass in 0..self.passes {- self.pass.encode(- input,- &mut histograms,- &mut prefixes,- &mut masks,- &mut ranks,- &mut done,- rows as u32,- k,- pass,- command_buffer,- );- }- self.collect.encode(- input,- output_ids,- output_scores,- &prefixes,- &mut counts,- &mut done,- &mut keys,- rows as u32,- k,- command_buffer,- );+ assert!(rows > 0 && columns > 0 && k > 0 && k <= MAX_K && k <= columns);+ let context = command_buffer.context.clone();+ let kernel = self.kernels[usize::from(sorted)].get_or_init(|| {+ RadixTopKSmallMetalKernel::new(&context, sorted).expect("Failed to create radix top-k kernel")+ });+ kernel.encode(input, output_ids, output_scores, rows, columns, k, command_buffer);Ok(())}}diff --git a/crates/uzu-engine/src/backends/webgpu/kernel.rs b/crates/uzu-engine/src/backends/webgpu/kernel.rsindex c7c4a2129..11a5382f2 100644--- a/crates/uzu-engine/src/backends/webgpu/kernel.rs+++ b/crates/uzu-engine/src/backends/webgpu/kernel.rs@@ -92,10 +92,7 @@ impl MatmulKernel for Unsupported<WebGPU> {impl RadixTopKSmall for Unsupported<WebGPU> {type Backend = WebGPU;- fn new(- _context: &WebGPUContext,- _columns: u32,- ) -> Result<Self, WebGPUError> {+ fn new(_context: &WebGPUContext) -> Result<Self, WebGPUError> {Err(WebGPUError::NotSupported)}@@ -105,7 +102,9 @@ impl RadixTopKSmall for Unsupported<WebGPU> {_output_ids: impl BufferMut<Backend = WebGPU>,_output_scores: impl BufferMut<Backend = WebGPU>,_rows: u32,+ _columns: u32,_k: u32,+ _sorted: bool,_command_buffer: &mut WebGPUCommandBufferEncoding,) -> Result<(), WebGPUError> {Err(WebGPUError::NotSupported)diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex 8499e59e5..714f95595 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -197,7 +197,6 @@ pub enum WeaverEncodeError<B: Backend> {impl<B: Backend> Weaver<B> {pub async fn new(- vocab_size: u32,config: &WeaverConfig,readout_input_hadamard_factors: Option<Arc<B::GlobalGpuBuffer>>,parameter_tree: &ParameterTree<'_, B>,@@ -321,8 +320,7 @@ impl<B: Backend> Weaver<B> {}).transpose().map_err(WeaverNewError::Backend)?;- let top_k =- <B::Kernels as Kernels>::RadixTopKSmall::new(context, vocab_size).map_err(WeaverNewError::Backend)?;+ let top_k = <B::Kernels as Kernels>::RadixTopKSmall::new(context).map_err(WeaverNewError::Backend)?;let top_children = <B::Kernels as Kernels>::WeaverSelectKernel::new(context, 256, Children, false).map_err(WeaverNewError::Backend)?;let top_children_with_prune_noise =@@ -687,7 +685,9 @@ impl<B: Backend> Weaver<B> {&mut candidate_ids,&mut candidate_logits,pool_depth_count,+ vocab_size,self.candidate_pool_size,+ true,command_buffer,).map_err(WeaverEncodeError::Backend)?;diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex fccbe2d7c..812ebe301 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -132,7 +132,6 @@ impl<B: Backend> DFlashTfmSpeculator<B> {let weaver = if let Some(weaver_config) = &config.weaver_config {Some(Weaver::new(- config.draft_config.vocab_size,weaver_config,target_embedding.readout_input_hadamard_factors(),&speculator_tree.subtree("weaver"),diff --git a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rsindex ee9a5d2ca..10708a306 100644--- a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs@@ -33,14 +33,17 @@ fn radix_top_k_small<B: Backend>(rows: usize,columns: usize,k: usize,+ sorted: bool,) -> (Vec<u32>, Vec<f32>) {let context = create_context::<B>();let input = create_buffer_with_data::<B, bf16>(&context, input);let mut ids = create_buffer::<B, u32>(&context, rows * k);let mut scores = create_buffer::<B, f32>(&context, rows * k);- let kernel = <B::Kernels as Kernels>::RadixTopKSmall::new(&context, columns as u32).unwrap();+ let kernel = <B::Kernels as Kernels>::RadixTopKSmall::new(&context).unwrap();let mut command_buffer = context.create_command_buffer(None, None).unwrap();- kernel.encode(&input, &mut ids, &mut scores, rows as u32, k as u32, &mut command_buffer).unwrap();+ kernel+ .encode(&input, &mut ids, &mut scores, rows as u32, columns as u32, k as u32, sorted, &mut command_buffer)+ .unwrap();block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();(buffer_to_vec(&*context, &ids), buffer_to_vec(&*context, &scores))}@@ -50,13 +53,18 @@ fn reference(rows: usize,columns: usize,k: usize,+ sorted: bool,) -> (Vec<u32>, Vec<f32>) {let mut ids = Vec::with_capacity(rows * k);let mut scores = Vec::with_capacity(rows * k);for values in input.chunks_exact(columns) {let mut row = (0..columns).collect::<Vec<_>>();row.sort_unstable_by(|&left, &right| values[right].total_cmp(&values[left]).then_with(|| left.cmp(&right)));- for index in row.into_iter().take(k) {+ row.truncate(k);+ if !sorted {+ row.sort_unstable();+ }+ for index in row {ids.push(index as u32);scores.push(values[index]);}@@ -84,6 +92,7 @@ fn radix_top_k_small_matches_cpu() {(3, 1025, 1),(3, 1025, TARGET_K - 1),(3, 1025, TARGET_K),+ (1, 262_145, TARGET_K),(15, TARGET_COLUMNS, TARGET_K),] {let mut input = values(rows, columns).into_iter().map(bf16::from_f32).collect::<Vec<_>>();@@ -99,17 +108,20 @@ fn radix_top_k_small_matches_cpu() {];let special_count = special.len().min(input.len());input[..special_count].copy_from_slice(&special[..special_count]);- let expected = reference(&input.iter().map(|value| value.to_f32()).collect::<Vec<_>>(), rows, columns, k);- assert_output(&radix_top_k_small::<Cpu>(&input, rows, columns, k), &expected, shape);- for_each_non_cpu_backend!(|B| {- let actual = radix_top_k_small::<B>(&input, rows, columns, k);- assert_output(&actual, &expected, shape);- if columns == TARGET_COLUMNS {- for _ in 0..2 {- assert_output(&radix_top_k_small::<B>(&input, rows, columns, k), &expected, shape);+ for sorted in [true, false] {+ let expected =+ reference(&input.iter().map(|value| value.to_f32()).collect::<Vec<_>>(), rows, columns, k, sorted);+ assert_output(&radix_top_k_small::<Cpu>(&input, rows, columns, k, sorted), &expected, shape);+ for_each_non_cpu_backend!(|B| {+ let actual = radix_top_k_small::<B>(&input, rows, columns, k, sorted);+ assert_output(&actual, &expected, shape);+ if columns == TARGET_COLUMNS {+ for _ in 0..2 {+ assert_output(&radix_top_k_small::<B>(&input, rows, columns, k, sorted), &expected, shape);+ }}- }- });+ });+ }}}@@ -128,13 +140,23 @@ fn benchmark_radix_top_k_small() {);let mut ids = create_buffer::<Metal, u32>(&context, ROWS * TARGET_K);let mut scores = create_buffer::<Metal, f32>(&context, ROWS * TARGET_K);- let kernel =- <<Metal as Backend>::Kernels as Kernels>::RadixTopKSmall::new(&context, TARGET_COLUMNS as u32).unwrap();+ let kernel = <<Metal as Backend>::Kernels as Kernels>::RadixTopKSmall::new(&context).unwrap();let mut run = || {let start = Instant::now();let mut command_buffer = context.create_command_buffer(None, None).unwrap();for _ in 0..BATCH {- kernel.encode(&input, &mut ids, &mut scores, ROWS as u32, TARGET_K as u32, &mut command_buffer).unwrap();+ kernel+ .encode(+ &input,+ &mut ids,+ &mut scores,+ ROWS as u32,+ TARGET_COLUMNS as u32,+ TARGET_K as u32,+ true,+ &mut command_buffer,+ )+ .unwrap();}let completed = block_on(command_buffer.end_encoding().submit().wait_until_completed()).unwrap();(completed.gpu_execution_time().unwrap().div_f64(BATCH as f64), start.elapsed().div_f64(BATCH as f64))