A high-performance inference engine for AI models
Use BF16 logits for DFlash readout and remove widened output (#859)
Chen-Chen Yeh5df819608277
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rsindex f5f4825e6..be9089f4c 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs@@ -1,3 +1,5 @@+use half::bf16;+use crate::{backends::{common::{@@ -40,7 +42,7 @@ impl RadixTopKSmall for CpuRadixTopKSmall {let k = k as usize;assert!(rows > 0 && k > 0 && k <= MAX_K as usize && k <= columns);let input = input.as_buffer_range_ref();- let input = SendPtr(unsafe { (&*input.buffer().get()).as_ptr().add(input.range().start).cast::<f32>() });+ let input = SendPtr(unsafe { (&*input.buffer().get()).as_ptr().add(input.range().start).cast::<bf16>() });let output_ids = output_ids.as_buffer_range_mut();let output_ids = SendPtrMut(unsafe {(&mut *output_ids.buffer().get()).as_mut_ptr().add(output_ids.range().start).cast::<u32>()@@ -50,9 +52,12 @@ impl RadixTopKSmall for CpuRadixTopKSmall {(&mut *output_scores.buffer().get()).as_mut_ptr().add(output_scores.range().start).cast::<f32>()});encoder.as_command_buffer_mut().push_command(move || {- let values = unsafe { std::slice::from_raw_parts(input.as_ptr(), rows * columns) };let output_ids = unsafe { std::slice::from_raw_parts_mut(output_ids.as_ptr(), rows * k) };let output_scores = unsafe { std::slice::from_raw_parts_mut(output_scores.as_ptr(), rows * k) };+ let values = unsafe { std::slice::from_raw_parts(input.as_ptr(), rows * columns) }+ .iter()+ .map(|value| value.to_f32())+ .collect::<Vec<_>>();for (row, values) in values.chunks_exact(columns).enumerate() {let mut indices = (0..columns).collect::<Vec<_>>();let compare = |&left: &usize, &right: &usize| {diff --git a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metalindex 55b059d3e..90501bbb0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal@@ -28,7 +28,7 @@ METAL_FUNC void reset_arrival(device atomic_uint* count) {template <typename Visitor>METAL_FUNC void visit_partition_keys(- const device float* input,+ const device bfloat* input,uint columns,uint index_bits,uint partition,@@ -40,26 +40,26 @@ METAL_FUNC void visit_partition_keys(const uint begin = columns * partition / partitions;const uint end = columns * (partition + 1u) / partitions;for (uint column = begin + lid; column < end; column += THREADS_PER_TG)- visit(top_k_ordered_key(input[column], column, index_bits));+ visit(top_k_ordered_key(float(input[column]), column, index_bits));return;}- const device float4* input4 = reinterpret_cast<const device float4*>(input);+ const device vec<bfloat, VECTOR_WIDTH>* input4 = reinterpret_cast<const device vec<bfloat, VECTOR_WIDTH>*>(input);const uint vector_columns = columns / VECTOR_WIDTH;const uint vector_begin = vector_columns * partition / partitions;const uint vector_end = vector_columns * (partition + 1u) / partitions;for (uint vector = vector_begin + lid; vector < vector_end; vector += THREADS_PER_TG) {- const float4 values = input4[vector];+ const vec<bfloat, VECTOR_WIDTH> values = input4[vector];for (uint lane = 0; lane < VECTOR_WIDTH; ++lane)- visit(top_k_ordered_key(values[lane], vector * VECTOR_WIDTH + lane, index_bits));+ visit(top_k_ordered_key(float(values[lane]), vector * VECTOR_WIDTH + lane, index_bits));}if (partition + 1 == partitions) {for (uint column = vector_columns * VECTOR_WIDTH + lid; column < columns; column += THREADS_PER_TG)- visit(top_k_ordered_key(input[column], column, index_bits));+ visit(top_k_ordered_key(float(input[column]), column, index_bits));}}KERNEL(RadixTopKSmallPass)(- const device float* input,+ const device bfloat* input,device atomic_uint* partial_histograms,device ulong* prefixes,device ulong* prefix_masks,@@ -81,7 +81,7 @@ KERNEL(RadixTopKSmallPass)(const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);const uint passes = (32u + index_bits + RADIX_BITS - 1u) / RADIX_BITS;const uint shift = (passes - pass - 1u) * RADIX_BITS;- const device float* row_input = input + ulong(row) * columns;+ const device bfloat* row_input = input + ulong(row) * columns;const ulong prefix = prefixes[row];const ulong mask = prefix_masks[row];@@ -152,7 +152,7 @@ KERNEL(RadixTopKSmallPass)(}KERNEL(RadixTopKSmallCollect)(- const device float* input,+ const device bfloat* input,device uint* output_ids,device float* output_scores,const device ulong* prefixes,@@ -171,7 +171,7 @@ KERNEL(RadixTopKSmallCollect)(const uint row = group / partitions;const uint partition = group % partitions;const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);- const device float* row_input = input + ulong(row) * columns;+ const device bfloat* row_input = input + ulong(row) * columns;const ulong prefix = prefixes[row];visit_partition_keys(row_input, columns, index_bits, partition, partitions, lid, [&](ulong key) {diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rsindex 6a145194f..ca3c22849 100644--- a/crates/uzu-engine/src/encodable_block/decoder.rs+++ b/crates/uzu-engine/src/encodable_block/decoder.rs@@ -181,7 +181,9 @@ impl<B: Backend> Decoder<B> {Some(self.embedding.encode_readout(output_range.end - output_range.start,output,- self.embedding.data_type(),+ self.embedding.vocab_size(),+ None,+ true,encoder,)?)} else {diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex 6ae7f489b..59f268cec 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -334,7 +334,14 @@ impl<B: Backend> DFlash<B> {let mut lookahead_hidden =encoder.allocate_scratch(lookahead_rows.len()).map_err(DFlashEncodeError::Backend)?;encoder.encode_copy(&draft_hidden, lookahead_rows, &mut lookahead_hidden, ..);- let logits = target_embedding.encode_readout_raw(batch_size - 1, &lookahead_hidden, DataType::F32, encoder)?;+ let logits = target_embedding.encode_readout(+ batch_size - 1,+ &lookahead_hidden,+ target_embedding.vocab_size(),+ None,+ false,+ encoder,+ )?;encoder.pop_debug_group();diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rsindex c210bf60c..71c5bccf0 100644--- a/crates/uzu-engine/src/encodable_block/embedding.rs+++ b/crates/uzu-engine/src/encodable_block/embedding.rs@@ -74,14 +74,9 @@ struct LogitTransform<B: Backend> {scale: f32,soft_cap: Option<f32>,kernel: <B::Kernels as Kernels>::LogitTransformKernel,- widened_kernel: Option<<B::Kernels as Kernels>::LogitTransformKernel>,}impl<B: Backend> Embedding<B> {- pub fn data_type(&self) -> DataType {- self.data_type- }-pub fn vocab_size(&self) -> u32 {self.vocab_size}@@ -307,23 +302,10 @@ impl<B: Backend> Embedding<B> {let kernel =<B::Kernels as Kernels>::LogitTransformKernel::new(context, data_type, logit_soft_cap.is_some()).map_err(EmbeddingError::BackendError)?;- let widened_kernel = if data_type != DataType::F32 {- Some(- <B::Kernels as Kernels>::LogitTransformKernel::new(- context,- DataType::F32,- logit_soft_cap.is_some(),- )- .map_err(EmbeddingError::BackendError)?,- )- } else {- None- };Some(LogitTransform {scale: logit_scale,soft_cap: logit_soft_cap,kernel,- widened_kernel,})} else {None@@ -375,46 +357,17 @@ impl<B: Backend> Embedding<B> {&self,batch_dim: u32,input_allocation: &Allocation<B>,- output_data_type: DataType,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, EmbeddingError<B>> {- let mut output_allocation = self.encode_readout_raw(batch_dim, input_allocation, output_data_type, encoder)?;- let native_output = output_data_type == self.data_type;-- if let Some(logit_transform) = &self.logit_transform {- let length = batch_dim * self.vocab_size;- let kernel = if native_output {- &logit_transform.kernel- } else {- assert_eq!(output_data_type, DataType::F32, "unsupported readout output data type");- logit_transform.widened_kernel.as_ref().expect("widened logit transform kernel is missing")- };- kernel.encode(- &mut output_allocation,- length,- logit_transform.scale,- logit_transform.soft_cap.unwrap_or(0.0),- encoder,- );- }-- Ok(output_allocation)- }-- pub fn encode_readout_raw(- &self,- batch_dim: u32,- input_allocation: &Allocation<B>,- output_data_type: DataType,+ output_dim: u32,+ gather_indices: Option<&Allocation<B>>,+ apply_logit_transform: bool,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, EmbeddingError<B>> {encoder.push_debug_group("embedding readout");- assert!(batch_dim > 0, "Embedding readout requires at least one row");- let native_output = output_data_type == self.data_type;+ assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");let input_hadamard = self.readout_input_hadamard();- let mut output_allocation = encoder- .allocate_scratch_for_shape(&[batch_dim, self.vocab_size], output_data_type)+ let mut output = encoder+ .allocate_scratch_for_shape(&[batch_dim, output_dim], self.data_type).map_err(EmbeddingError::BackendError)?;let (matrix, readout) = self.readout_operands();@@ -443,95 +396,31 @@ impl<B: Backend> Embedding<B> {b: matrix.matmul_b(),b_leading_dimension: None,b_transpose: true,- d: &mut output_allocation,+ d: &mut output,d_transform: MatmulDOps::none(),- gather_indices: None,+ gather_indices,m: batch_dim,- n: self.vocab_size,+ n: output_dim,k: self.model_dim,};- if native_output {- readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;- } else {- let mut widened = <B::Kernels as Kernels>::MatmulKernel::new(- encoder.context(),- self.data_type,- self.data_type,- output_data_type,- )- .map_err(EmbeddingError::BackendError)?;- widened.encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;- }-- encoder.pop_debug_group();-- Ok(output_allocation)- }-- pub fn encode_readout_sparse_raw(- &self,- input: &Allocation<B>,- token_ids: &Allocation<B>,- rows: u32,- ids_per_row: u32,- encoder: &mut Encoder<B>,- ) -> Result<Allocation<B>, EmbeddingError<B>> {- encoder.push_debug_group("embedding readout (sparse)");-- assert!(rows > 0 && ids_per_row > 0);- let input_hadamard = self.readout_input_hadamard();- let (matrix, readout) = self.readout_operands();- let b = matrix.matmul_b();-- let mut output = encoder- .allocate_scratch_for_shape(&[rows, ids_per_row], self.data_type)- .map_err(EmbeddingError::BackendError)?;-- let mut rht_input: Option<Allocation<B>> = None;- let a = match input_hadamard {- Some(input_hadamard) => {- let mut transformed = encoder.allocate_scratch(input.size()).map_err(EmbeddingError::BackendError)?;- input_hadamard.kernel.encode_fp(- input,- &mut transformed,- &input_hadamard.factors,- rows,- self.model_dim,- encoder,- );- rht_input.insert(transformed)- },- None => input,- };+ readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;- readout- .lock()- .encode(- MatmulArguments {- a: MatmulA::FullPrecision {- values: a,- offset: 0,- },- b,- b_leading_dimension: None,- b_transpose: true,- d: &mut output,- d_transform: MatmulDOps::none(),- gather_indices: Some(token_ids),- m: rows,- n: ids_per_row,- k: self.model_dim,- },+ if apply_logit_transform && let Some(logit_transform) = &self.logit_transform {+ let length = batch_dim * output_dim;+ logit_transform.kernel.encode(+ &mut output,+ length,+ logit_transform.scale,+ logit_transform.soft_cap.unwrap_or(0.0),encoder,- )- .map_err(EmbeddingError::BackendError)?;+ );+ }encoder.pop_debug_group();Ok(output)}}-fn readout_kernel<B: Backend>(context: &B::Context,data_type: DataType,diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex 427821f4f..309fc4697 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -474,11 +474,12 @@ impl<B: Backend> Weaver<B> {.readout_query_projection.encode(normalized_output, batch_node_count, encoder).map_err(WeaverEncodeError::Backend)?;- let logit_residuals = target_embedding.encode_readout_sparse_raw(- &query,- batch_candidate_ids,+ let logit_residuals = target_embedding.encode_readout(batch_node_count,+ &query,self.candidate_pool_size,+ Some(batch_candidate_ids),+ false,encoder,)?;let mut child_token_ids = encoder@@ -556,7 +557,7 @@ impl<B: Backend> Weaver<B> {// lookahead row form the candidate pool node expansions draw from.let vocab_size = target_embedding.vocab_size();assert!(- logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DataType::F32),+ logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DATA_TYPE),"draft logits do not cover the lookahead rows");let mut candidate_ids = encoderdiff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rsindex f18c31edf..2ce2a3fee 100644--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs@@ -128,7 +128,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {weight_loader.tree().assert_all_tensors_validated()?;- let sampling = Sampling::new(DataType::F32, config.draft_config.vocab_size);+ let sampling = Sampling::new(data_type, config.draft_config.vocab_size);Ok(Some(Self {context,diff --git a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rsindex 7fd6c7d27..3645c9779 100644--- a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs@@ -1,6 +1,7 @@#[cfg(backend = "metal")]use std::time::Instant;+use half::bf16;use uzu_engine_macros::uzu_test;#[cfg(backend = "metal")]@@ -26,13 +27,13 @@ fn values(}fn radix_top_k_small<B: Backend>(- input: &[f32],+ input: &[bf16],rows: usize,columns: usize,k: usize,) -> (Vec<u32>, Vec<f32>) {let context = create_context::<B>();- let input = alloc_allocation_with_data::<B, f32>(&context, input);+ let input = alloc_allocation_with_data::<B, bf16>(&context, input);let mut ids = alloc_allocation::<B, u32>(&context, rows * k);let mut scores = alloc_allocation::<B, f32>(&context, rows * k);let kernel = <B::Kernels as Kernels>::RadixTopKSmall::new(&context, columns as u32).unwrap();@@ -83,20 +84,20 @@ fn radix_top_k_small_matches_cpu() {(3, 1025, TARGET_K),(15, TARGET_COLUMNS, TARGET_K),] {- let mut input = values(rows, columns);+ let mut input = values(rows, columns).into_iter().map(bf16::from_f32).collect::<Vec<_>>();let special = [- f32::INFINITY,- f32::NEG_INFINITY,- -0.0,- 0.0,- 1.0,- 1.0,- f32::from_bits(0x7fc0_0001),- f32::from_bits(0xffc0_0001),+ bf16::INFINITY,+ bf16::NEG_INFINITY,+ bf16::NEG_ZERO,+ bf16::ZERO,+ bf16::ONE,+ bf16::ONE,+ bf16::from_bits(0x7fc1),+ bf16::from_bits(0xffc1),];let special_count = special.len().min(input.len());input[..special_count].copy_from_slice(&special[..special_count]);- let expected = reference(&input, rows, columns, k);+ let expected = reference(&input.iter().map(|value| value.to_f32()).collect::<Vec<_>>(), rows, columns, k);assert_output(&radix_top_k_small::<Cpu>(&input, rows, columns, k), &expected, shape);for_each_non_cpu_backend!(|B| {let actual = radix_top_k_small::<B>(&input, rows, columns, k);@@ -119,7 +120,10 @@ fn benchmark_radix_top_k_small() {const BATCH: u32 = 16;let context = create_context::<Metal>();- let input = alloc_allocation_with_data::<Metal, f32>(&context, &values(ROWS, TARGET_COLUMNS));+ let input = alloc_allocation_with_data::<Metal, bf16>(+ &context,+ &values(ROWS, TARGET_COLUMNS).into_iter().map(bf16::from_f32).collect::<Vec<_>>(),+ );let mut ids = alloc_allocation::<Metal, u32>(&context, ROWS * TARGET_K);let mut scores = alloc_allocation::<Metal, f32>(&context, ROWS * TARGET_K);let kernel =