Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Use BF16 logits for DFlash readout and remove widened output (#859)

Chen-Chen Yeh5df819608277
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs
index f5f4825e6..be9089f4c 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/radix_top_k_small.rs
@@ -1,3 +1,5 @@
+use half::bf16;
+
use crate::{
backends::{
common::{
@@ -40,7 +42,7 @@ impl RadixTopKSmall for CpuRadixTopKSmall {
let k = k as usize;
assert!(rows > 0 && k > 0 && k <= MAX_K as usize && k <= columns);
let input = input.as_buffer_range_ref();
- let input = SendPtr(unsafe { (&*input.buffer().get()).as_ptr().add(input.range().start).cast::<f32>() });
+ let input = SendPtr(unsafe { (&*input.buffer().get()).as_ptr().add(input.range().start).cast::<bf16>() });
let output_ids = output_ids.as_buffer_range_mut();
let output_ids = SendPtrMut(unsafe {
(&mut *output_ids.buffer().get()).as_mut_ptr().add(output_ids.range().start).cast::<u32>()
@@ -50,9 +52,12 @@ impl RadixTopKSmall for CpuRadixTopKSmall {
(&mut *output_scores.buffer().get()).as_mut_ptr().add(output_scores.range().start).cast::<f32>()
});
encoder.as_command_buffer_mut().push_command(move || {
- let values = unsafe { std::slice::from_raw_parts(input.as_ptr(), rows * columns) };
let output_ids = unsafe { std::slice::from_raw_parts_mut(output_ids.as_ptr(), rows * k) };
let output_scores = unsafe { std::slice::from_raw_parts_mut(output_scores.as_ptr(), rows * k) };
+ let values = unsafe { std::slice::from_raw_parts(input.as_ptr(), rows * columns) }
+ .iter()
+ .map(|value| value.to_f32())
+ .collect::<Vec<_>>();
for (row, values) in values.chunks_exact(columns).enumerate() {
let mut indices = (0..columns).collect::<Vec<_>>();
let compare = |&left: &usize, &right: &usize| {
diff --git a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal
index 55b059d3e..90501bbb0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/radix_top_k_small.metal
@@ -28,7 +28,7 @@ METAL_FUNC void reset_arrival(device atomic_uint* count) {
template <typename Visitor>
METAL_FUNC void visit_partition_keys(
- const device float* input,
+ const device bfloat* input,
uint columns,
uint index_bits,
uint partition,
@@ -40,26 +40,26 @@ METAL_FUNC void visit_partition_keys(
const uint begin = columns * partition / partitions;
const uint end = columns * (partition + 1u) / partitions;
for (uint column = begin + lid; column < end; column += THREADS_PER_TG)
- visit(top_k_ordered_key(input[column], column, index_bits));
+ visit(top_k_ordered_key(float(input[column]), column, index_bits));
return;
}
- const device float4* input4 = reinterpret_cast<const device float4*>(input);
+ const device vec<bfloat, VECTOR_WIDTH>* input4 = reinterpret_cast<const device vec<bfloat, VECTOR_WIDTH>*>(input);
const uint vector_columns = columns / VECTOR_WIDTH;
const uint vector_begin = vector_columns * partition / partitions;
const uint vector_end = vector_columns * (partition + 1u) / partitions;
for (uint vector = vector_begin + lid; vector < vector_end; vector += THREADS_PER_TG) {
- const float4 values = input4[vector];
+ const vec<bfloat, VECTOR_WIDTH> values = input4[vector];
for (uint lane = 0; lane < VECTOR_WIDTH; ++lane)
- visit(top_k_ordered_key(values[lane], vector * VECTOR_WIDTH + lane, index_bits));
+ visit(top_k_ordered_key(float(values[lane]), vector * VECTOR_WIDTH + lane, index_bits));
}
if (partition + 1 == partitions) {
for (uint column = vector_columns * VECTOR_WIDTH + lid; column < columns; column += THREADS_PER_TG)
- visit(top_k_ordered_key(input[column], column, index_bits));
+ visit(top_k_ordered_key(float(input[column]), column, index_bits));
}
}
KERNEL(RadixTopKSmallPass)(
- const device float* input,
+ const device bfloat* input,
device atomic_uint* partial_histograms,
device ulong* prefixes,
device ulong* prefix_masks,
@@ -81,7 +81,7 @@ KERNEL(RadixTopKSmallPass)(
const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);
const uint passes = (32u + index_bits + RADIX_BITS - 1u) / RADIX_BITS;
const uint shift = (passes - pass - 1u) * RADIX_BITS;
- const device float* row_input = input + ulong(row) * columns;
+ const device bfloat* row_input = input + ulong(row) * columns;
const ulong prefix = prefixes[row];
const ulong mask = prefix_masks[row];
@@ -152,7 +152,7 @@ KERNEL(RadixTopKSmallPass)(
}
KERNEL(RadixTopKSmallCollect)(
- const device float* input,
+ const device bfloat* input,
device uint* output_ids,
device float* output_scores,
const device ulong* prefixes,
@@ -171,7 +171,7 @@ KERNEL(RadixTopKSmallCollect)(
const uint row = group / partitions;
const uint partition = group % partitions;
const uint index_bits = columns <= 1 ? 1u : 32u - clz(columns - 1u);
- const device float* row_input = input + ulong(row) * columns;
+ const device bfloat* row_input = input + ulong(row) * columns;
const ulong prefix = prefixes[row];
visit_partition_keys(row_input, columns, index_bits, partition, partitions, lid, [&](ulong key) {
diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rs
index 6a145194f..ca3c22849 100644
--- a/crates/uzu-engine/src/encodable_block/decoder.rs
+++ b/crates/uzu-engine/src/encodable_block/decoder.rs
@@ -181,7 +181,9 @@ impl<B: Backend> Decoder<B> {
Some(self.embedding.encode_readout(
output_range.end - output_range.start,
output,
- self.embedding.data_type(),
+ self.embedding.vocab_size(),
+ None,
+ true,
encoder,
)?)
} else {
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index 6ae7f489b..59f268cec 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -334,7 +334,14 @@ impl<B: Backend> DFlash<B> {
let mut lookahead_hidden =
encoder.allocate_scratch(lookahead_rows.len()).map_err(DFlashEncodeError::Backend)?;
encoder.encode_copy(&draft_hidden, lookahead_rows, &mut lookahead_hidden, ..);
- let logits = target_embedding.encode_readout_raw(batch_size - 1, &lookahead_hidden, DataType::F32, encoder)?;
+ let logits = target_embedding.encode_readout(
+ batch_size - 1,
+ &lookahead_hidden,
+ target_embedding.vocab_size(),
+ None,
+ false,
+ encoder,
+ )?;
encoder.pop_debug_group();
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index c210bf60c..71c5bccf0 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -74,14 +74,9 @@ struct LogitTransform<B: Backend> {
scale: f32,
soft_cap: Option<f32>,
kernel: <B::Kernels as Kernels>::LogitTransformKernel,
- widened_kernel: Option<<B::Kernels as Kernels>::LogitTransformKernel>,
}
impl<B: Backend> Embedding<B> {
- pub fn data_type(&self) -> DataType {
- self.data_type
- }
-
pub fn vocab_size(&self) -> u32 {
self.vocab_size
}
@@ -307,23 +302,10 @@ impl<B: Backend> Embedding<B> {
let kernel =
<B::Kernels as Kernels>::LogitTransformKernel::new(context, data_type, logit_soft_cap.is_some())
.map_err(EmbeddingError::BackendError)?;
- let widened_kernel = if data_type != DataType::F32 {
- Some(
- <B::Kernels as Kernels>::LogitTransformKernel::new(
- context,
- DataType::F32,
- logit_soft_cap.is_some(),
- )
- .map_err(EmbeddingError::BackendError)?,
- )
- } else {
- None
- };
Some(LogitTransform {
scale: logit_scale,
soft_cap: logit_soft_cap,
kernel,
- widened_kernel,
})
} else {
None
@@ -375,46 +357,17 @@ impl<B: Backend> Embedding<B> {
&self,
batch_dim: u32,
input_allocation: &Allocation<B>,
- output_data_type: DataType,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, EmbeddingError<B>> {
- let mut output_allocation = self.encode_readout_raw(batch_dim, input_allocation, output_data_type, encoder)?;
- let native_output = output_data_type == self.data_type;
-
- if let Some(logit_transform) = &self.logit_transform {
- let length = batch_dim * self.vocab_size;
- let kernel = if native_output {
- &logit_transform.kernel
- } else {
- assert_eq!(output_data_type, DataType::F32, "unsupported readout output data type");
- logit_transform.widened_kernel.as_ref().expect("widened logit transform kernel is missing")
- };
- kernel.encode(
- &mut output_allocation,
- length,
- logit_transform.scale,
- logit_transform.soft_cap.unwrap_or(0.0),
- encoder,
- );
- }
-
- Ok(output_allocation)
- }
-
- pub fn encode_readout_raw(
- &self,
- batch_dim: u32,
- input_allocation: &Allocation<B>,
- output_data_type: DataType,
+ output_dim: u32,
+ gather_indices: Option<&Allocation<B>>,
+ apply_logit_transform: bool,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, EmbeddingError<B>> {
encoder.push_debug_group("embedding readout");
- assert!(batch_dim > 0, "Embedding readout requires at least one row");
- let native_output = output_data_type == self.data_type;
+ assert!(batch_dim > 0 && output_dim > 0, "Embedding readout requires non-empty dimensions");
let input_hadamard = self.readout_input_hadamard();
- let mut output_allocation = encoder
- .allocate_scratch_for_shape(&[batch_dim, self.vocab_size], output_data_type)
+ let mut output = encoder
+ .allocate_scratch_for_shape(&[batch_dim, output_dim], self.data_type)
.map_err(EmbeddingError::BackendError)?;
let (matrix, readout) = self.readout_operands();
@@ -443,95 +396,31 @@ impl<B: Backend> Embedding<B> {
b: matrix.matmul_b(),
b_leading_dimension: None,
b_transpose: true,
- d: &mut output_allocation,
+ d: &mut output,
d_transform: MatmulDOps::none(),
- gather_indices: None,
+ gather_indices,
m: batch_dim,
- n: self.vocab_size,
+ n: output_dim,
k: self.model_dim,
};
- if native_output {
- readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
- } else {
- let mut widened = <B::Kernels as Kernels>::MatmulKernel::new(
- encoder.context(),
- self.data_type,
- self.data_type,
- output_data_type,
- )
- .map_err(EmbeddingError::BackendError)?;
- widened.encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
- }
-
- encoder.pop_debug_group();
-
- Ok(output_allocation)
- }
-
- pub fn encode_readout_sparse_raw(
- &self,
- input: &Allocation<B>,
- token_ids: &Allocation<B>,
- rows: u32,
- ids_per_row: u32,
- encoder: &mut Encoder<B>,
- ) -> Result<Allocation<B>, EmbeddingError<B>> {
- encoder.push_debug_group("embedding readout (sparse)");
-
- assert!(rows > 0 && ids_per_row > 0);
- let input_hadamard = self.readout_input_hadamard();
- let (matrix, readout) = self.readout_operands();
- let b = matrix.matmul_b();
-
- let mut output = encoder
- .allocate_scratch_for_shape(&[rows, ids_per_row], self.data_type)
- .map_err(EmbeddingError::BackendError)?;
-
- let mut rht_input: Option<Allocation<B>> = None;
- let a = match input_hadamard {
- Some(input_hadamard) => {
- let mut transformed = encoder.allocate_scratch(input.size()).map_err(EmbeddingError::BackendError)?;
- input_hadamard.kernel.encode_fp(
- input,
- &mut transformed,
- &input_hadamard.factors,
- rows,
- self.model_dim,
- encoder,
- );
- rht_input.insert(transformed)
- },
- None => input,
- };
+ readout.lock().encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
- readout
- .lock()
- .encode(
- MatmulArguments {
- a: MatmulA::FullPrecision {
- values: a,
- offset: 0,
- },
- b,
- b_leading_dimension: None,
- b_transpose: true,
- d: &mut output,
- d_transform: MatmulDOps::none(),
- gather_indices: Some(token_ids),
- m: rows,
- n: ids_per_row,
- k: self.model_dim,
- },
+ if apply_logit_transform && let Some(logit_transform) = &self.logit_transform {
+ let length = batch_dim * output_dim;
+ logit_transform.kernel.encode(
+ &mut output,
+ length,
+ logit_transform.scale,
+ logit_transform.soft_cap.unwrap_or(0.0),
encoder,
- )
- .map_err(EmbeddingError::BackendError)?;
+ );
+ }
encoder.pop_debug_group();
Ok(output)
}
}
-
fn readout_kernel<B: Backend>(
context: &B::Context,
data_type: DataType,
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index 427821f4f..309fc4697 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -474,11 +474,12 @@ impl<B: Backend> Weaver<B> {
.readout_query_projection
.encode(normalized_output, batch_node_count, encoder)
.map_err(WeaverEncodeError::Backend)?;
- let logit_residuals = target_embedding.encode_readout_sparse_raw(
- &query,
- batch_candidate_ids,
+ let logit_residuals = target_embedding.encode_readout(
batch_node_count,
+ &query,
self.candidate_pool_size,
+ Some(batch_candidate_ids),
+ false,
encoder,
)?;
let mut child_token_ids = encoder
@@ -556,7 +557,7 @@ impl<B: Backend> Weaver<B> {
// lookahead row form the candidate pool node expansions draw from.
let vocab_size = target_embedding.vocab_size();
assert!(
- logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DataType::F32),
+ logits.size() >= size_for_shape(&[pool_depth_count, vocab_size], DATA_TYPE),
"draft logits do not cover the lookahead rows"
);
let mut candidate_ids = encoder
diff --git a/crates/uzu-engine/src/speculators/dflash_tfm.rs b/crates/uzu-engine/src/speculators/dflash_tfm.rs
index f18c31edf..2ce2a3fee 100644
--- a/crates/uzu-engine/src/speculators/dflash_tfm.rs
+++ b/crates/uzu-engine/src/speculators/dflash_tfm.rs
@@ -128,7 +128,7 @@ impl<B: Backend> DFlashTfmSpeculator<B> {
weight_loader.tree().assert_all_tensors_validated()?;
- let sampling = Sampling::new(DataType::F32, config.draft_config.vocab_size);
+ let sampling = Sampling::new(data_type, config.draft_config.vocab_size);
Ok(Some(Self {
context,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs
index 7fd6c7d27..3645c9779 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/radix_top_k_small_test.rs
@@ -1,6 +1,7 @@
#[cfg(backend = "metal")]
use std::time::Instant;
+use half::bf16;
use uzu_engine_macros::uzu_test;
#[cfg(backend = "metal")]
@@ -26,13 +27,13 @@ fn values(
}
fn radix_top_k_small<B: Backend>(
- input: &[f32],
+ input: &[bf16],
rows: usize,
columns: usize,
k: usize,
) -> (Vec<u32>, Vec<f32>) {
let context = create_context::<B>();
- let input = alloc_allocation_with_data::<B, f32>(&context, input);
+ let input = alloc_allocation_with_data::<B, bf16>(&context, input);
let mut ids = alloc_allocation::<B, u32>(&context, rows * k);
let mut scores = alloc_allocation::<B, f32>(&context, rows * k);
let kernel = <B::Kernels as Kernels>::RadixTopKSmall::new(&context, columns as u32).unwrap();
@@ -83,20 +84,20 @@ fn radix_top_k_small_matches_cpu() {
(3, 1025, TARGET_K),
(15, TARGET_COLUMNS, TARGET_K),
] {
- let mut input = values(rows, columns);
+ let mut input = values(rows, columns).into_iter().map(bf16::from_f32).collect::<Vec<_>>();
let special = [
- f32::INFINITY,
- f32::NEG_INFINITY,
- -0.0,
- 0.0,
- 1.0,
- 1.0,
- f32::from_bits(0x7fc0_0001),
- f32::from_bits(0xffc0_0001),
+ bf16::INFINITY,
+ bf16::NEG_INFINITY,
+ bf16::NEG_ZERO,
+ bf16::ZERO,
+ bf16::ONE,
+ bf16::ONE,
+ bf16::from_bits(0x7fc1),
+ bf16::from_bits(0xffc1),
];
let special_count = special.len().min(input.len());
input[..special_count].copy_from_slice(&special[..special_count]);
- let expected = reference(&input, rows, columns, k);
+ let expected = reference(&input.iter().map(|value| value.to_f32()).collect::<Vec<_>>(), rows, columns, k);
assert_output(&radix_top_k_small::<Cpu>(&input, rows, columns, k), &expected, shape);
for_each_non_cpu_backend!(|B| {
let actual = radix_top_k_small::<B>(&input, rows, columns, k);
@@ -119,7 +120,10 @@ fn benchmark_radix_top_k_small() {
const BATCH: u32 = 16;
let context = create_context::<Metal>();
- let input = alloc_allocation_with_data::<Metal, f32>(&context, &values(ROWS, TARGET_COLUMNS));
+ let input = alloc_allocation_with_data::<Metal, bf16>(
+ &context,
+ &values(ROWS, TARGET_COLUMNS).into_iter().map(bf16::from_f32).collect::<Vec<_>>(),
+ );
let mut ids = alloc_allocation::<Metal, u32>(&context, ROWS * TARGET_K);
let mut scores = alloc_allocation::<Metal, f32>(&context, ROWS * TARGET_K);
let kernel =