Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Migration to allocations (#349)

LuckyIYI40708e01a9e4
diff --git a/crates/backend-uzu/src/array/array.rs b/crates/backend-uzu/src/array/array.rs
index 77d169e1f..bdfe6a44f 100644
--- a/crates/backend-uzu/src/array/array.rs
+++ b/crates/backend-uzu/src/array/array.rs
@@ -1,60 +1,21 @@
-use std::{cell::RefCell, rc::Rc};
+use std::fmt;
use crate::{
DataType,
array::size_for_shape,
- backends::common::{AsBufferRangeMut, Backend, Buffer},
+ backends::common::{
+ Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, BufferRangeMut, BufferRangeRef,
+ },
};
-#[derive(Debug)]
-pub struct Array<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>> = <B as Backend>::DenseBuffer> {
- pub(super) buffer: Rc<RefCell<Buf>>,
+pub struct Array<B: Backend, BufferRange: AsBufferRangeMut<Buffer: Buffer<Backend = B>> = Allocation<B>> {
+ pub(super) buffer_range: BufferRange,
pub(super) offset: usize,
pub(super) shape: Box<[usize]>,
pub(super) data_type: DataType,
}
-impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, Buf> {
- // Constructors
- pub unsafe fn from_parts(
- buffer: Rc<RefCell<Buf>>,
- offset: usize,
- shape: &[usize],
- data_type: DataType,
- ) -> Self {
- let required_bytes = size_for_shape(shape, data_type);
- let buffer_range = buffer.borrow().as_buffer_range_ref().range();
- let buffer_range_size = buffer_range.end - buffer_range.start;
-
- assert!(
- offset + required_bytes <= buffer_range_size,
- "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",
- shape,
- data_type,
- offset,
- offset + required_bytes,
- buffer_range_size
- );
- Self {
- buffer: buffer.clone(),
- offset,
- shape: shape.into(),
- data_type,
- }
- }
-
- pub fn view(
- &self,
- shape: &[usize],
- ) -> Self {
- unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }
- }
-
- // Getters
- pub fn buffer(&self) -> Rc<RefCell<Buf>> {
- self.buffer.clone()
- }
-
+impl<B: Backend, BufferRange: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, BufferRange> {
pub fn offset(&self) -> usize {
self.offset
}
@@ -74,15 +35,65 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, Bu
pub fn num_elements(&self) -> usize {
self.shape.iter().product()
}
+
+ pub fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, BufferRange::Buffer> {
+ self.buffer_range.as_buffer_range_ref().subrange(self.offset..self.offset + self.size())
+ }
+
+ pub fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, BufferRange::Buffer> {
+ let offset = self.offset;
+ let size = self.size();
+ self.buffer_range.as_buffer_range_mut().subrange(offset..offset + size)
+ }
}
-impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Clone for Array<B, Buf> {
- fn clone(&self) -> Self {
+impl<B: Backend> Array<B, Allocation<B>> {
+ pub unsafe fn from_allocation(
+ allocation: Allocation<B>,
+ offset: usize,
+ shape: &[usize],
+ data_type: DataType,
+ ) -> Self {
+ let required_bytes = size_for_shape(shape, data_type);
+ let allocation_size = allocation.as_buffer_range_ref().range().len();
+ assert!(offset + required_bytes <= allocation_size, "Array shape exceeds allocation length",);
Self {
- buffer: self.buffer.clone(),
- offset: self.offset,
- shape: self.shape.clone(),
- data_type: self.data_type,
+ buffer_range: allocation,
+ offset,
+ shape: shape.into(),
+ data_type,
}
}
+
+ pub fn allocation(&self) -> &Allocation<B> {
+ &self.buffer_range
+ }
+
+ pub fn allocation_mut(&mut self) -> &mut Allocation<B> {
+ &mut self.buffer_range
+ }
+
+ pub fn into_allocation(self) -> Allocation<B> {
+ assert_eq!(self.offset, 0, "Array view cannot be converted into Allocation");
+ assert_eq!(
+ size_for_shape(&self.shape, self.data_type),
+ self.buffer_range.as_buffer_range_ref().range().len(),
+ "Partial Array view cannot be converted into Allocation",
+ );
+ self.buffer_range
+ }
+}
+
+impl<B: Backend, BufferRange: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> fmt::Debug for Array<B, BufferRange> {
+ fn fmt(
+ &self,
+ formatter: &mut fmt::Formatter<'_>,
+ ) -> fmt::Result {
+ formatter
+ .debug_struct("Array")
+ .field("offset", &self.offset)
+ .field("shape", &self.shape)
+ .field("data_type", &self.data_type)
+ .finish()
+ }
}
diff --git a/crates/backend-uzu/src/array/context_ext.rs b/crates/backend-uzu/src/array/context_ext.rs
index 3b80227f0..7fa543aaf 100644
--- a/crates/backend-uzu/src/array/context_ext.rs
+++ b/crates/backend-uzu/src/array/context_ext.rs
@@ -1,9 +1,7 @@
-use std::{cell::RefCell, rc::Rc};
-
use crate::{
Array, ArrayElement, DataType,
array::size_for_shape,
- backends::common::{Backend, Buffer, Context},
+ backends::common::{AllocationType, Context},
};
pub trait ArrayContextExt: Context {
@@ -11,23 +9,18 @@ pub trait ArrayContextExt: Context {
&self,
shape: &[usize],
data_type: DataType,
- label: &str,
- ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {
- let buffer_size_bytes = size_for_shape(shape, data_type);
-
- let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");
- buffer.set_label(Some(label));
-
- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }
+ ) -> Array<Self::Backend> {
+ let size = size_for_shape(shape, data_type);
+ let allocation = self.create_allocation(size, AllocationType::Global).expect("Failed to create allocation");
+ unsafe { Array::from_allocation(allocation, 0, shape, data_type) }
}
fn create_array_zeros(
&self,
shape: &[usize],
data_type: DataType,
- label: &str,
- ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {
- let mut array = self.create_array_uninitialized(shape, data_type, label);
+ ) -> Array<Self::Backend> {
+ let mut array = self.create_array_uninitialized(shape, data_type);
array.as_bytes_mut().fill(0);
array
}
@@ -36,8 +29,7 @@ pub trait ArrayContextExt: Context {
&self,
shape: &[usize],
data: &[T],
- label: &str,
- ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {
+ ) -> Array<Self::Backend> {
let size_from_shape: usize = shape.iter().product();
assert_eq!(
data.len(),
@@ -47,7 +39,7 @@ pub trait ArrayContextExt: Context {
data.len()
);
- let mut array = self.create_array_uninitialized(shape, T::data_type(), label);
+ let mut array = self.create_array_uninitialized(shape, T::data_type());
array.as_slice_mut().copy_from_slice(data);
array
}
diff --git a/crates/backend-uzu/src/array/dense.rs b/crates/backend-uzu/src/array/dense.rs
index 1deda3906..45ac031d2 100644
--- a/crates/backend-uzu/src/array/dense.rs
+++ b/crates/backend-uzu/src/array/dense.rs
@@ -3,18 +3,15 @@ use std::{ops::Range, os::raw::c_void, ptr::NonNull};
use ndarray::{ArrayView, Dimension, IxDyn};
use crate::{
- ArrayElement,
- array::Array,
+ Array, ArrayElement,
backends::common::{AsBufferRangeMut, Backend, DenseBuffer},
};
-impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<B, Buf> {
+impl<B: Backend, BufferRange: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<B, BufferRange> {
pub fn cpu_ptr(&self) -> NonNull<c_void> {
- let buffer = self.buffer.borrow();
- let buffer_range = buffer.as_buffer_range_ref();
+ let buffer_range = self.as_buffer_range_ref();
let range = buffer_range.range();
-
- unsafe { buffer_range.buffer().cpu_ptr().add(range.start + self.offset) }
+ unsafe { buffer_range.buffer().cpu_ptr().add(range.start) }
}
pub fn as_bytes(&self) -> &[u8] {
@@ -22,7 +19,12 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<
}
pub fn as_bytes_mut(&mut self) -> &mut [u8] {
- unsafe { std::slice::from_raw_parts_mut(self.cpu_ptr().as_ptr() as *mut u8, self.size()) }
+ let size = self.size();
+ let buffer_range = self.as_buffer_range_mut();
+ let range = buffer_range.range();
+ unsafe {
+ std::slice::from_raw_parts_mut((buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start), size)
+ }
}
pub fn as_slice<T: ArrayElement>(&self) -> &[T] {
@@ -41,66 +43,60 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<
pub fn copy_from_array<
OtherBackend: Backend,
- OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,
+ OtherBufferRange: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,
>(
&mut self,
- other: &Array<OtherBackend, OtherBuf>,
+ other: &Array<OtherBackend, OtherBufferRange>,
) {
- assert_eq!(self.shape, other.shape);
- assert_eq!(self.data_type, other.data_type);
+ assert_eq!(self.shape(), other.shape());
+ assert_eq!(self.data_type(), other.data_type());
self.as_bytes_mut().copy_from_slice(other.as_bytes());
}
pub fn copy_slice<
OtherBackend: Backend,
- OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,
+ OtherBufferRange: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,
>(
&mut self,
- source: &Array<OtherBackend, OtherBuf>,
+ source: &Array<OtherBackend, OtherBufferRange>,
axis: usize,
src_range: Range<usize>,
dst_offset: usize,
) {
assert_eq!(self.shape().len(), source.shape().len(), "Rank mismatch");
assert!(axis < self.shape().len(), "Axis out of bounds");
- for (i, (a, b)) in self.shape().iter().zip(source.shape()).enumerate() {
- if i != axis {
- assert_eq!(a, b, "Shapes must match on all non-sliced axes");
+ for (dimension_index, (destination_dim, source_dim)) in self.shape().iter().zip(source.shape()).enumerate() {
+ if dimension_index != axis {
+ assert_eq!(destination_dim, source_dim, "Shapes must match on all non-sliced axes");
}
}
assert_eq!(self.data_type(), source.data_type(), "Arrays must have the same data type");
- let elem_size = self.data_type().size_in_bytes();
-
- // The number of contiguous elements to copy for each slice operation
- let block_size_elems = self.shape().iter().skip(axis + 1).product::<usize>();
- let block_size_bytes = block_size_elems * elem_size;
-
- // The total number of blocks to copy
+ let element_size = self.data_type().size_in_bytes();
+ let block_size_elements = self.shape().iter().skip(axis + 1).product::<usize>();
+ let block_size_bytes = block_size_elements * element_size;
let num_blocks: usize = self.shape().iter().take(axis).product();
-
- // Strides between the start of each block
- let src_stride_bytes = source.shape()[axis] * block_size_bytes;
- let dst_stride_bytes = self.shape()[axis] * block_size_bytes;
+ let source_stride_bytes = source.shape()[axis] * block_size_bytes;
+ let destination_stride_bytes = self.shape()[axis] * block_size_bytes;
let rows_to_copy = src_range.end - src_range.start;
assert!(dst_offset + rows_to_copy <= self.shape()[axis]);
assert!(src_range.end <= source.shape()[axis]);
- let src_buf = source.as_bytes();
- let dst_buf = self.as_bytes_mut();
+ let source_bytes = source.as_bytes();
+ let destination_bytes = self.as_bytes_mut();
let copy_bytes = rows_to_copy * block_size_bytes;
- for i in 0..num_blocks {
- let src_block_start = i * src_stride_bytes;
- let dst_block_start = i * dst_stride_bytes;
+ for block_index in 0..num_blocks {
+ let source_block_start = block_index * source_stride_bytes;
+ let destination_block_start = block_index * destination_stride_bytes;
- let src_start = src_block_start + src_range.start * block_size_bytes;
- let dst_start = dst_block_start + dst_offset * block_size_bytes;
+ let source_start = source_block_start + src_range.start * block_size_bytes;
+ let destination_start = destination_block_start + dst_offset * block_size_bytes;
- let src_slice = &src_buf[src_start..src_start + copy_bytes];
- let dst_slice = &mut dst_buf[dst_start..dst_start + copy_bytes];
- dst_slice.copy_from_slice(src_slice);
+ let source_slice = &source_bytes[source_start..source_start + copy_bytes];
+ let destination_slice = &mut destination_bytes[destination_start..destination_start + copy_bytes];
+ destination_slice.copy_from_slice(source_slice);
}
}
@@ -110,15 +106,15 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<
) {
assert_eq!(self.data_type(), T::data_type());
- let dst_slice = self.as_slice_mut::<T>();
+ let destination_slice = self.as_slice_mut::<T>();
- if let Some(src_slice) = view.as_slice_memory_order() {
- assert!(src_slice.len() <= dst_slice.len());
- dst_slice[..src_slice.len()].copy_from_slice(src_slice);
+ if let Some(source_slice) = view.as_slice_memory_order() {
+ assert!(source_slice.len() <= destination_slice.len());
+ destination_slice[..source_slice.len()].copy_from_slice(source_slice);
} else {
- assert!(view.len() <= dst_slice.len());
- for (d, s) in dst_slice.iter_mut().zip(view.iter()) {
- *d = *s;
+ assert!(view.len() <= destination_slice.len());
+ for (destination, source) in destination_slice.iter_mut().zip(view.iter()) {
+ *destination = *source;
}
}
}
@@ -126,10 +122,10 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<
fn validate_element_type<T: ArrayElement>(&self) {
assert_eq!(
T::data_type(),
- self.data_type,
+ self.data_type(),
"Invalid data type, expected {:?}, actual {:?}",
T::data_type(),
- self.data_type
+ self.data_type()
);
}
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
index 48b657f4e..54ee95c0e 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs
@@ -1,5 +1,5 @@
use std::{
- cell::{Cell, RefCell},
+ cell::RefCell,
collections::{BTreeMap, HashMap},
fs::File,
path::Path,
@@ -9,8 +9,8 @@ use std::{
use serde::Deserialize;
use crate::{
- DataType,
- array::{Array, ArrayContextExt, size_for_shape},
+ DataType, allocation_to_vec,
+ array::{ArrayContextExt, size_for_shape},
audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid},
backends::common::{
Backend, Context, Encoder, Kernels, Pending,
@@ -26,11 +26,7 @@ use crate::{
EmbeddingConfigCommon, InnerModelConfig, TtsAudioDecoderConfig, TtsConfig,
},
encodable_block::{Decoder, EncodingParameters, LayerExecutables, RMSNorm},
- forward_pass::{
- model_shape::ModelShape,
- scratch_buffers::ScratchBuffers,
- state::{ArrayId, ForwardPassState, SharedBuffers},
- },
+ forward_pass::{model_shape::ModelShape, state::SharedBuffers},
parameters::ParameterLoader,
};
@@ -324,22 +320,14 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
let kernel = <B::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32)
.map_err(|err| AudioError::Runtime(format!("failed to initialize fsq decode kernel: {err}")))?;
- let mut tokens_array = context.create_array_zeros(
- &[batch_size, self.config.num_groups(), frames],
- DataType::I32,
- "nanocodec_fsq_decode_tokens",
- );
- tokens_array.as_slice_mut::<i32>().copy_from_slice(&tokens_i32);
+ let tokens_allocation =
+ context.create_array_from(&[batch_size, self.config.num_groups(), frames], &tokens_i32).into_allocation();
- let mut lengths_array =
- context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");
- lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
+ let lengths_allocation = context.create_array_from(&[batch_size], &lengths_i32).into_allocation();
- let output = context.create_array_zeros(
- &[batch_size, self.config.channels(), frames],
- DataType::F32,
- "nanocodec_fsq_decode_output",
- );
+ let mut output = context
+ .create_array_uninitialized(&[batch_size, self.config.channels(), frames], DataType::F32)
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref())
.map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;
@@ -348,27 +336,18 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
let frames_i32 = usize_to_i32(frames, "frames")?;
let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;
let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;
- {
- let tokens_buffer = tokens_array.buffer();
- let tokens_buffer = tokens_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
-
- kernel.encode(
- &*tokens_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
- num_groups_i32,
- frames_i32,
- codebook_dim_i32,
- self.config.num_levels_per_group(),
- self.config.dim_base_index(),
- batch_size_i32,
- &mut encoder,
- );
- }
+ kernel.encode(
+ &tokens_allocation,
+ &mut output,
+ &lengths_allocation,
+ num_groups_i32,
+ frames_i32,
+ codebook_dim_i32,
+ self.config.num_levels_per_group(),
+ self.config.dim_base_index(),
+ batch_size_i32,
+ &mut encoder,
+ );
encoder
.end_encoding()
@@ -377,7 +356,7 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
.map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ decode command buffer: {err}")))?;
Ok(DecodedPaddedAudio {
- samples: output.as_slice::<f32>().to_vec(),
+ samples: allocation_to_vec::<B, f32>(&output),
channels: self.config.channels(),
frames,
lengths: lengths_usize,
@@ -415,21 +394,14 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
let kernel = <B::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32)
.map_err(|err| AudioError::Runtime(format!("failed to initialize fsq encode kernel: {err}")))?;
- let mut input = context.create_array_zeros(
- &[batch_size, self.config.channels(), frames],
- DataType::F32,
- "nanocodec_fsq_encode_input",
- );
- input.as_slice_mut::<f32>().copy_from_slice(&padded_input);
+ let input =
+ context.create_array_from(&[batch_size, self.config.channels(), frames], &padded_input).into_allocation();
- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");
- lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
+ let lengths = context.create_array_from(&[batch_size], &lengths_i32).into_allocation();
- let tokens = context.create_array_zeros(
- &[batch_size, self.config.num_groups(), frames],
- DataType::I32,
- "nanocodec_fsq_encode_tokens",
- );
+ let mut tokens = context
+ .create_array_uninitialized(&[batch_size, self.config.num_groups(), frames], DataType::I32)
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref())
.map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;
@@ -438,28 +410,19 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
let frames_i32 = usize_to_i32(frames, "frames")?;
let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;
let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;
- {
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let tokens_buffer = tokens.buffer();
- let mut tokens_buffer = tokens_buffer.borrow_mut();
- let lengths_buffer = lengths.buffer();
- let lengths_buffer = lengths_buffer.borrow();
-
- kernel.encode(
- &*input_buffer,
- &mut *tokens_buffer,
- &*lengths_buffer,
- num_groups_i32,
- frames_i32,
- codebook_dim_i32,
- self.config.num_levels_per_group(),
- self.config.dim_base_index(),
- self.config.eps(),
- batch_size_i32,
- &mut encoder,
- );
- }
+ kernel.encode(
+ &input,
+ &mut tokens,
+ &lengths,
+ num_groups_i32,
+ frames_i32,
+ codebook_dim_i32,
+ self.config.num_levels_per_group(),
+ self.config.dim_base_index(),
+ self.config.eps(),
+ batch_size_i32,
+ &mut encoder,
+ );
encoder
.end_encoding()
@@ -467,8 +430,9 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
.wait_until_completed()
.map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ encode command buffer: {err}")))?;
- let mut tokens_u32 = vec![0_u32; tokens.num_elements()];
- for (index, &token) in tokens.as_slice::<i32>().iter().enumerate() {
+ let encoded_tokens = allocation_to_vec::<B, i32>(&tokens);
+ let mut tokens_u32 = vec![0_u32; encoded_tokens.len()];
+ for (index, &token) in encoded_tokens.iter().enumerate() {
if token < 0 {
return Err(AudioError::Runtime(format!(
"fsq encode returned negative token at index {index}: {token}"
@@ -539,8 +503,9 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
pub fn end_decode_stream(
&self,
- _state: AudioDecodeStreamState,
+ state: AudioDecodeStreamState,
) -> AudioResult<()> {
+ drop(state);
Ok(())
}
@@ -766,11 +731,6 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
codebooks,
window_frames,
)?;
- // The scratch buffers inside the workspace were encoded into the
- // submitted command buffer. Reset the workspace so the next decode
- // pass allocates fresh buffers instead of reusing ones the in-flight
- // command buffer may still be reading.
- resources.reset_for_pending();
let audio_offset_frames = window_start
.checked_mul(fishaudio.upsample_factor)
.ok_or(AudioError::Runtime("stream audio offset overflow".to_string()))?;
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rs
index 8c19f5df3..6970aa9cf 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rs
@@ -1,44 +1,76 @@
use std::sync::mpsc::Receiver;
use super::*;
+use crate::{allocation_to_vec, array::Array, backends::common::Allocation};
-pub struct SubmittedDecodedPaddedAudio<B: Backend> {
- pub(in crate::audio::nanocodec::runtime) output: Array<B>,
- pub(in crate::audio::nanocodec::runtime) channels: usize,
- pub(in crate::audio::nanocodec::runtime) frames: usize,
- pub(in crate::audio::nanocodec::runtime) lengths: Vec<usize>,
- pub(in crate::audio::nanocodec::runtime) final_command_buffer: Option<Pending<B>>,
- pub(in crate::audio::nanocodec::runtime) completion_notification: Option<Receiver<()>>,
+pub enum SubmittedDecodedPaddedAudio<B: Backend> {
+ Ready(DecodedPaddedAudio),
+ Pending {
+ output: Allocation<B>,
+ retained_inputs: Box<[Array<B>]>,
+ data_type: DataType,
+ channels: usize,
+ frames: usize,
+ lengths: Vec<usize>,
+ final_command_buffer: Pending<B>,
+ completion_notification: Receiver<()>,
+ },
}
impl<B: Backend> SubmittedDecodedPaddedAudio<B> {
pub(crate) fn is_complete(&self) -> bool {
use std::sync::mpsc::TryRecvError;
- self.completion_notification.as_ref().is_none_or(|notification| match notification.try_recv() {
- Ok(()) | Err(TryRecvError::Disconnected) => true,
- Err(TryRecvError::Empty) => false,
- })
+ match self {
+ Self::Ready(_) => true,
+ Self::Pending {
+ completion_notification,
+ ..
+ } => match completion_notification.try_recv() {
+ Ok(()) | Err(TryRecvError::Disconnected) => true,
+ Err(TryRecvError::Empty) => false,
+ },
+ }
}
- pub(in crate::audio::nanocodec::runtime) fn resolve(mut self) -> AudioResult<DecodedPaddedAudio> {
- if let Some(command_buffer) = self.final_command_buffer.take() {
- command_buffer.wait_until_completed().map_err(|err| {
- AudioError::Runtime(format!("failed to wait for FishAudio decoder command buffer: {err}"))
- })?;
+ pub(in crate::audio::nanocodec::runtime) fn resolve(self) -> AudioResult<DecodedPaddedAudio> {
+ match self {
+ Self::Ready(decoded) => Ok(decoded),
+ Self::Pending {
+ output,
+ retained_inputs,
+ data_type,
+ channels,
+ frames,
+ lengths,
+ final_command_buffer,
+ completion_notification: _,
+ } => {
+ let completed_command_buffer = final_command_buffer.wait_until_completed().map_err(|err| {
+ AudioError::Runtime(format!("failed to wait for FishAudio decoder command buffer: {err}"))
+ })?;
+ drop(retained_inputs);
+ let samples_result: AudioResult<Vec<f32>> = match data_type {
+ DataType::F32 => Ok(allocation_to_vec::<B, f32>(&output)),
+ DataType::F16 => {
+ Ok(allocation_to_vec::<B, half::f16>(&output).iter().map(|&value| f32::from(value)).collect())
+ },
+ DataType::BF16 => {
+ Ok(allocation_to_vec::<B, half::bf16>(&output).iter().map(|&value| f32::from(value)).collect())
+ },
+ dt => Err(AudioError::Runtime(format!("unsupported vocoder output dtype: {dt:?}"))),
+ };
+ drop(output);
+ drop(completed_command_buffer);
+ let samples = samples_result?;
+ Ok(DecodedPaddedAudio {
+ samples,
+ channels,
+ frames,
+ lengths,
+ })
+ },
}
- let samples: Vec<f32> = match self.output.data_type() {
- DataType::F32 => self.output.as_slice::<f32>().to_vec(),
- DataType::F16 => self.output.as_slice::<half::f16>().iter().map(|&v| f32::from(v)).collect(),
- DataType::BF16 => self.output.as_slice::<half::bf16>().iter().map(|&v| f32::from(v)).collect(),
- dt => return Err(AudioError::Runtime(format!("unsupported vocoder output dtype: {dt:?}"))),
- };
- Ok(DecodedPaddedAudio {
- samples,
- channels: self.channels,
- frames: self.frames,
- lengths: self.lengths,
- })
}
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
index 3e2d90914..cc0b16259 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
@@ -1,9 +1,10 @@
use super::*;
+use crate::array::{Array, ArrayContextExt};
pub(super) type BackendParameterTree<'loader, B> = crate::parameters::ParameterTree<'loader, <B as Backend>::Context>;
pub(super) fn read_float_array<B: Backend, const RANK: usize>(
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
name: &str,
expected_data_type: DataType,
) -> AudioResult<([usize; RANK], Array<B>)> {
@@ -16,8 +17,8 @@ pub(super) fn read_float_array<B: Backend, const RANK: usize>(
}
if array.shape().len() != RANK {
return Err(AudioError::Runtime(format!(
- "expected rank-{RANK} tensor for '{name}', got shape {:?}",
- array.shape()
+ "expected rank-{RANK} tensor for '{name}', got rank {}",
+ array.shape().len()
)));
}
let mut dims = [0usize; RANK];
@@ -26,7 +27,7 @@ pub(super) fn read_float_array<B: Backend, const RANK: usize>(
}
pub(super) fn read_float_vector_exact<B: Backend>(
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
name: &str,
expected_len: usize,
expected_data_type: DataType,
@@ -42,7 +43,7 @@ pub(super) fn read_float_vector_exact<B: Backend>(
}
pub(super) fn read_float_matrix_exact<B: Backend>(
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
name: &str,
expected_rows: usize,
expected_cols: usize,
@@ -58,21 +59,29 @@ pub(super) fn read_float_matrix_exact<B: Backend>(
Ok(array)
}
-pub(super) fn outer_axis_view<B: Backend>(
- array: &Array<B>,
+pub(super) fn copy_to_outer_axis_slice<B: Backend>(
+ destination: &mut Array<B>,
index: usize,
+ source: &Array<B>,
slice_shape: &[usize],
-) -> AudioResult<Array<B>> {
- let slice_bytes = size_for_shape(slice_shape, array.data_type());
- let offset = index
- .checked_mul(slice_bytes)
- .and_then(|value| value.checked_add(array.offset()))
- .ok_or(AudioError::Runtime("array outer-axis view offset overflow".to_string()))?;
- Ok(unsafe { Array::from_parts(array.buffer(), offset, slice_shape, array.data_type()) })
+) -> AudioResult<()> {
+ let slice_bytes = size_for_shape(slice_shape, source.data_type());
+ let offset =
+ index.checked_mul(slice_bytes).ok_or(AudioError::Runtime("outer-axis slice offset overflow".to_string()))?;
+ let end =
+ offset.checked_add(slice_bytes).ok_or(AudioError::Runtime("outer-axis slice end overflow".to_string()))?;
+ if end > destination.size() {
+ return Err(AudioError::Runtime(format!(
+ "outer-axis slice exceeds destination: end {end}, destination size {}",
+ destination.size()
+ )));
+ }
+ destination.as_bytes_mut()[offset..end].copy_from_slice(source.as_bytes());
+ Ok(())
}
pub(super) fn read_conv1d_layer<B: Backend>(
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
data_type: DataType,
dilation: usize,
groups: usize,
@@ -100,7 +109,7 @@ pub(super) fn read_conv1d_layer<B: Backend>(
}
pub(super) fn read_conv_transpose1d_layer<B: Backend>(
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
data_type: DataType,
stride: usize,
groups: usize,
@@ -139,7 +148,7 @@ pub(super) fn read_conv_transpose1d_layer<B: Backend>(
}
pub(super) fn read_pointwise_conv_layer<B: Backend>(
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
data_type: DataType,
expected_out_dim: usize,
expected_in_dim: usize,
@@ -147,7 +156,7 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(
let weight = read_float_matrix_exact::<B>(tree, "weights", expected_out_dim, expected_in_dim, data_type)?;
let bias = read_float_vector_exact::<B>(tree, "biases", expected_out_dim, data_type)?;
Ok(StructuredAudioPointwiseConv {
- weight: weight.view(&[expected_out_dim, expected_in_dim, 1]),
+ weight,
bias,
cin: expected_in_dim,
cout: expected_out_dim,
@@ -156,19 +165,18 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(
pub(super) fn read_norm_layer<B: Backend>(
context: &Rc<B::Context>,
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
data_type: DataType,
channels: usize,
epsilon: f32,
subtract_mean: bool,
use_bias: bool,
- label_prefix: &str,
) -> AudioResult<StructuredAudioNorm<B>> {
let scales = read_float_vector_exact::<B>(tree, "scales", channels, data_type)?;
let bias = if use_bias {
read_float_vector_exact::<B>(tree, "biases", channels, data_type)?
} else {
- context.create_array_zeros(&[channels], data_type, &format!("{label_prefix}_bias"))
+ context.create_array_zeros(&[channels], data_type)
};
Ok(StructuredAudioNorm {
scales,
@@ -180,7 +188,7 @@ pub(super) fn read_norm_layer<B: Backend>(
pub(super) fn read_convnext_layer<B: Backend>(
context: &Rc<B::Context>,
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
data_type: DataType,
norm_config: &DescriptAudioConvNeXtNormConfig,
) -> AudioResult<StructuredAudioConvNeXt<B>> {
@@ -201,13 +209,6 @@ pub(super) fn read_convnext_layer<B: Backend>(
dilation: 1,
groups: depthwise_shape[0],
};
- if depthwise_conv.bias.shape()[0] != depthwise_conv.cout {
- return Err(AudioError::Runtime(format!(
- "ConvNeXt depthwise bias mismatch at 'dwconv.biases': expected {}, got {}",
- depthwise_conv.cout,
- depthwise_conv.bias.shape()[0]
- )));
- }
if depthwise_conv.cout == 0 || depthwise_conv.kernel_size == 0 {
return Err(AudioError::InvalidTokenCardinality);
}
@@ -226,7 +227,6 @@ pub(super) fn read_convnext_layer<B: Backend>(
norm_config.epsilon,
norm_config.subtract_mean,
norm_config.use_bias,
- "structured_audio_convnext_norm",
)?;
let (pwconv1_shape, _) = read_float_array::<B, 2>(&tree.subtree("pwconv1")?, "weights", data_type)?;
if pwconv1_shape[1] != channels {
@@ -246,7 +246,7 @@ pub(super) fn read_convnext_layer<B: Backend>(
}
pub(super) fn read_residual_unit_layer<B: Backend>(
- tree: &BackendParameterTree<'_, B>,
+ tree: &BackendParameterTree<B>,
data_type: DataType,
dilation: usize,
) -> AudioResult<StructuredAudioResidualUnit<B>> {
@@ -270,13 +270,6 @@ pub(super) fn read_residual_unit_layer<B: Backend>(
}
alpha
};
- if snake1_alpha.shape()[0] != conv1.cin {
- return Err(AudioError::Runtime(format!(
- "residual snake1 alpha mismatch: expected {}, got {}",
- conv1.cin,
- snake1_alpha.shape()[0]
- )));
- }
let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, 1, 1)?;
Ok(StructuredAudioResidualUnit {
snake1_alpha,
@@ -288,7 +281,7 @@ pub(super) fn read_residual_unit_layer<B: Backend>(
pub(super) fn build_vocoder_graph_from_tree<B: Backend>(
context: &Rc<B::Context>,
- root: &BackendParameterTree<'_, B>,
+ root: &BackendParameterTree<B>,
config: &DescriptAudioCodecConfig,
data_type: DataType,
) -> AudioResult<StructuredAudioDecoderGraph<B>> {
@@ -330,11 +323,7 @@ pub(super) fn build_vocoder_graph_from_tree<B: Backend>(
let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1)?;
let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3)?;
let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9)?;
- if snake_alpha.shape()[0] != trans_conv.cin
- || res_unit1.conv1.cin != channels
- || res_unit2.conv1.cin != channels
- || res_unit3.conv1.cin != channels
- {
+ if res_unit1.conv1.cin != channels || res_unit2.conv1.cin != channels || res_unit3.conv1.cin != channels {
return Err(AudioError::Runtime(format!(
"structured audio decoder block {index} channel mismatch in exported weights"
)));
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rs
index 0c2ec5054..3d94a9f7f 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rs
@@ -1,4 +1,10 @@
use super::*;
+use crate::array::Array;
+
+pub(super) struct QuantizerDecodeOutput<B: Backend> {
+ pub(super) output: Array<B>,
+ pub(super) retained_inputs: Box<[Array<B>]>,
+}
impl StructuredAudioCodecGraph {
fn conv1d_input_context<B: Backend>(layer: &StructuredAudioConv1d<B>) -> AudioResult<usize> {
@@ -97,14 +103,13 @@ impl StructuredAudioCodecGraph {
pub(super) fn decode_quantizer_to_nsc_array_enqueued<B: Backend>(
&self,
resources: &StructuredAudioRuntimeResources<B>,
- context: &Rc<B::Context>,
encoder: &mut Encoder<B>,
tokens: &[u32],
lengths: &[usize],
batch_size: usize,
codebooks: usize,
frames: usize,
- ) -> AudioResult<Array<B>> {
+ ) -> AudioResult<QuantizerDecodeOutput<B>> {
if codebooks != self.total_codebooks {
return Err(AudioError::Runtime(format!(
"structured audio codebook mismatch: expected {}, got {codebooks}",
@@ -148,11 +153,24 @@ impl StructuredAudioCodecGraph {
let mut lengths_array = resources.length_staging(batch_size);
lengths_array.as_slice_mut::<i32>()[..lengths_i32.len()].copy_from_slice(&lengths_i32);
- let output = context.create_array_zeros(
- &[batch_size, frames, self.input_dim],
- quantizer_resources.data_type,
- "structured_audio_quantizer_output_nsc",
- );
+ let output = unsafe {
+ Array::from_allocation(
+ encoder
+ .allocate_scratch(size_for_shape(
+ &[batch_size, frames, self.input_dim],
+ quantizer_resources.data_type,
+ ))
+ .map_err(|err| {
+ AudioError::Runtime(format!("failed to allocate structured audio quantizer output: {err}"))
+ })?,
+ 0,
+ &[batch_size, frames, self.input_dim],
+ quantizer_resources.data_type,
+ )
+ };
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
let codebooks_i32 = usize_to_i32(codebooks, "codebooks")?;
let frames_i32 = usize_to_i32(frames, "frames")?;
@@ -162,34 +180,16 @@ impl StructuredAudioCodecGraph {
let semantic_cardinality_i32 = usize_to_i32(quantizer_resources.semantic_cardinality, "semantic_cardinality")?;
let residual_cardinality_i32 = usize_to_i32(quantizer_resources.residual_cardinality, "residual_cardinality")?;
- let tokens_buffer = tokens_array.buffer();
- let tokens_buffer = tokens_buffer.borrow();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
- let semantic_codebook_buffer = quantizer_resources.semantic_codebook.buffer();
- let semantic_codebook_buffer = semantic_codebook_buffer.borrow();
- let semantic_out_proj_buffer = quantizer_resources.semantic_out_proj.buffer();
- let semantic_out_proj_buffer = semantic_out_proj_buffer.borrow();
- let semantic_out_bias_buffer = quantizer_resources.semantic_out_bias.buffer();
- let semantic_out_bias_buffer = semantic_out_bias_buffer.borrow();
- let residual_codebooks_buffer = quantizer_resources.residual_codebooks.buffer();
- let residual_codebooks_buffer = residual_codebooks_buffer.borrow();
- let residual_out_proj_buffer = quantizer_resources.residual_out_proj.buffer();
- let residual_out_proj_buffer = residual_out_proj_buffer.borrow();
- let residual_out_bias_buffer = quantizer_resources.residual_out_bias.buffer();
- let residual_out_bias_buffer = residual_out_bias_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
quantizer_resources.kernel.encode(
- &*tokens_buffer,
- &*lengths_buffer,
- &*semantic_codebook_buffer,
- &*semantic_out_proj_buffer,
- &*semantic_out_bias_buffer,
- &*residual_codebooks_buffer,
- &*residual_out_proj_buffer,
- &*residual_out_bias_buffer,
- &mut *output_buffer,
+ tokens_array.allocation(),
+ lengths_array.allocation(),
+ quantizer_resources.semantic_codebook.allocation(),
+ quantizer_resources.semantic_out_proj.allocation(),
+ quantizer_resources.semantic_out_bias.allocation(),
+ quantizer_resources.residual_codebooks.allocation(),
+ quantizer_resources.residual_out_proj.allocation(),
+ quantizer_resources.residual_out_bias.allocation(),
+ &mut output,
batch_i32,
codebooks_i32,
frames_i32,
@@ -200,7 +200,11 @@ impl StructuredAudioCodecGraph {
residual_cardinality_i32,
encoder,
);
+ let retained_inputs = vec![tokens_array, lengths_array].into_boxed_slice();
- Ok(output)
+ Ok(QuantizerDecodeOutput {
+ output: unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) },
+ retained_inputs,
+ })
}
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rs
index c5b448575..08204c37c 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rs
@@ -1,7 +1,7 @@
use std::sync::mpsc::channel;
use super::*;
-use crate::audio::nanocodec::runtime::profile::SubmittedDecodedPaddedAudio;
+use crate::{array::Array, audio::nanocodec::runtime::profile::SubmittedDecodedPaddedAudio};
impl StructuredAudioCodecGraph {
fn run_residual_unit_enqueued<B: Backend>(
@@ -18,52 +18,56 @@ impl StructuredAudioCodecGraph {
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
let ws = &resources.decode_workspace;
- let ctx = resources.context();
- let data_type = input.data_type();
-
- let residual = input.clone();
- let mut x = snake1d_enqueue(
+ let res_snake1 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);
+ let x = snake1d_enqueue(
encoder,
input,
- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake1")?,
+ res_snake1,
&unit.snake1_alpha,
batch_size,
channels,
seq_len,
+ self.vocoder_data_type,
kernels,
)?;
- x = causal_conv1d_grouped_enqueue(
+ let res_conv1 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);
+ let x = causal_conv1d_grouped_enqueue(
encoder,
&x,
- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv1")?,
+ res_conv1,
&unit.conv1,
SequenceLayout::Ncs,
lengths,
lengths_array,
batch_size,
seq_len,
+ self.vocoder_data_type,
kernels,
)?;
- x = snake1d_enqueue(
+ let res_snake2 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);
+ let x = snake1d_enqueue(
encoder,
&x,
- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake2")?,
+ res_snake2,
&unit.snake2_alpha,
batch_size,
channels,
seq_len,
+ self.vocoder_data_type,
kernels,
)?;
+ let res_conv2 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);
causal_conv1d_grouped_residual_enqueue(
encoder,
&x,
- &residual,
- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv2")?,
+ input,
+ res_conv2,
&unit.conv2,
lengths,
lengths_array,
batch_size,
seq_len,
+ self.vocoder_data_type,
kernels,
)
}
@@ -86,15 +90,12 @@ impl StructuredAudioCodecGraph {
.ok_or(AudioError::Runtime("structured audio length scaling overflow".to_string()))
})
.collect::<AudioResult<Vec<_>>>()?;
- let context = resources.context().clone();
- return Ok(SubmittedDecodedPaddedAudio {
- output: context.create_array_zeros(&[0], DataType::F32, "structured_audio_empty_decode_output"),
+ return Ok(SubmittedDecodedPaddedAudio::Ready(DecodedPaddedAudio {
+ samples: Vec::new(),
channels: 1,
frames: out_lengths.iter().copied().max().unwrap_or(0),
lengths: out_lengths,
- final_command_buffer: None,
- completion_notification: None,
- });
+ }));
}
let mut lengths_i32 = lengths
@@ -113,34 +114,10 @@ impl StructuredAudioCodecGraph {
let vocoder_graph = self.vocoder_graph(resources)?;
let kernels = resources.kernels(self.vocoder_data_type)?;
- let worst_case_bytes = {
- let scratch_bytes = |channels: usize, frame_count: usize| -> usize {
- size_for_shape(&[batch_size, channels, frame_count], self.vocoder_data_type)
- };
- let mut max_bytes = 0usize;
- let mut current_frames = frames;
- for (trans_conv, convnext) in vocoder_graph.upsample_blocks.iter() {
- current_frames = current_frames.saturating_mul(trans_conv.stride);
- max_bytes = max_bytes.max(scratch_bytes(trans_conv.cout, current_frames));
- max_bytes = max_bytes.max(scratch_bytes(convnext.pwconv1.cout, current_frames));
- }
- let mut current_channels = vocoder_graph.first_conv.cout;
- max_bytes = max_bytes.max(scratch_bytes(current_channels, current_frames));
- for block in vocoder_graph.decoder_blocks.iter() {
- max_bytes = max_bytes.max(scratch_bytes(current_channels, current_frames));
- current_frames = current_frames.saturating_mul(block.trans_conv.stride);
- current_channels = block.trans_conv.cout;
- max_bytes = max_bytes.max(scratch_bytes(current_channels, current_frames));
- }
- max_bytes
- };
- ws.prime_for(context.as_ref(), worst_case_bytes)?;
-
let mut x;
let mut x_layout = SequenceLayout::Nsc;
let quantized_nsc = self.decode_quantizer_to_nsc_array_enqueued(
resources,
- &context,
&mut encoder,
tokens,
lengths,
@@ -148,12 +125,11 @@ impl StructuredAudioCodecGraph {
codebooks,
frames,
)?;
- let _ = codebooks;
+ let retained_inputs = quantized_nsc.retained_inputs;
x = self.apply_post_module_enqueued(
resources,
- &context,
&mut encoder,
- &quantized_nsc,
+ quantized_nsc.output,
lengths,
batch_size,
frames,
@@ -162,11 +138,9 @@ impl StructuredAudioCodecGraph {
let mut current_channels = self.input_dim;
let mut current_frames = frames;
let mut next_lengths_i32 = vec![0_i32; lengths_i32.len()];
- let mut lengths_array = ws.lengths_array(&context, 0, lengths_i32.len());
+ let mut lengths_array = ws.lengths_array(&mut encoder, lengths_i32.len());
lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
- let mut next_lengths_array = ws.lengths_array(&context, 1, lengths_i32.len());
-
- let data_type = x.data_type();
+ let mut next_lengths_array = ws.lengths_array(&mut encoder, lengths_i32.len());
for (block_index, (trans_conv, convnext)) in vocoder_graph.upsample_blocks.iter().enumerate() {
if trans_conv.cin != current_channels {
@@ -181,10 +155,12 @@ impl StructuredAudioCodecGraph {
scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, trans_conv.stride)?;
next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);
+ let up_tconv =
+ ws.next_scratch(&mut encoder, &[batch_size, trans_conv.cout, next_frames], self.vocoder_data_type);
x = causal_conv_transpose1d_causal_pad_enqueue(
&mut encoder,
&x,
- ws.next_scratch(&context, &[batch_size, trans_conv.cout, next_frames], data_type, "up_tconv")?,
+ up_tconv,
trans_conv,
&next_lengths_i32,
batch_size,
@@ -192,12 +168,13 @@ impl StructuredAudioCodecGraph {
next_frames,
x_layout,
&next_lengths_array,
+ self.vocoder_data_type,
&kernels,
)
.map_err(|err| {
AudioError::Runtime(format!(
"structured audio upsample block {block_index} transpose_conv failed: {err} (x_len={}, batch_size={}, cin={}, seq_len_in={}, seq_len_out={})",
- x.num_elements(),
+ x.size(),
batch_size,
trans_conv.cin,
current_frames,
@@ -234,21 +211,22 @@ impl StructuredAudioCodecGraph {
vocoder_graph.first_conv.cin, current_channels
)));
}
+ let dec_first_conv = ws.next_scratch(
+ &mut encoder,
+ &[batch_size, vocoder_graph.first_conv.cout, current_frames],
+ self.vocoder_data_type,
+ );
x = causal_conv1d_grouped_enqueue(
&mut encoder,
&x,
- ws.next_scratch(
- &context,
- &[batch_size, vocoder_graph.first_conv.cout, current_frames],
- data_type,
- "dec_first_conv",
- )?,
+ dec_first_conv,
&vocoder_graph.first_conv,
x_layout,
&lengths_i32,
&lengths_array,
batch_size,
current_frames,
+ self.vocoder_data_type,
&kernels,
)?;
current_channels = vocoder_graph.first_conv.cout;
@@ -260,14 +238,17 @@ impl StructuredAudioCodecGraph {
block.trans_conv.cin, current_channels
)));
}
+ let dec_snake =
+ ws.next_scratch(&mut encoder, &[batch_size, current_channels, current_frames], self.vocoder_data_type);
x = snake1d_enqueue(
&mut encoder,
&x,
- ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "dec_snake")?,
+ dec_snake,
&block.snake_alpha,
batch_size,
current_channels,
current_frames,
+ self.vocoder_data_type,
&kernels,
)?;
@@ -277,10 +258,15 @@ impl StructuredAudioCodecGraph {
scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, block.trans_conv.stride)?;
next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);
+ let dec_tconv = ws.next_scratch(
+ &mut encoder,
+ &[batch_size, block.trans_conv.cout, next_frames],
+ self.vocoder_data_type,
+ );
x = causal_conv_transpose1d_causal_pad_enqueue(
&mut encoder,
&x,
- ws.next_scratch(&context, &[batch_size, block.trans_conv.cout, next_frames], data_type, "dec_tconv")?,
+ dec_tconv,
&block.trans_conv,
&next_lengths_i32,
batch_size,
@@ -288,6 +274,7 @@ impl StructuredAudioCodecGraph {
next_frames,
SequenceLayout::Ncs,
&next_lengths_array,
+ self.vocoder_data_type,
&kernels,
)?;
@@ -334,41 +321,43 @@ impl StructuredAudioCodecGraph {
)?;
}
+ let final_snake =
+ ws.next_scratch(&mut encoder, &[batch_size, current_channels, current_frames], self.vocoder_data_type);
x = snake1d_enqueue(
&mut encoder,
&x,
- ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "final_snake")?,
+ final_snake,
&vocoder_graph.final_snake_alpha,
batch_size,
current_channels,
current_frames,
+ self.vocoder_data_type,
&kernels,
)?;
+ let final_conv = ws.next_scratch(
+ &mut encoder,
+ &[batch_size, vocoder_graph.final_conv.cout, current_frames],
+ self.vocoder_data_type,
+ );
x = causal_conv1d_grouped_enqueue(
&mut encoder,
&x,
- ws.next_scratch(
- &context,
- &[batch_size, vocoder_graph.final_conv.cout, current_frames],
- data_type,
- "final_conv",
- )?,
+ final_conv,
&vocoder_graph.final_conv,
SequenceLayout::Ncs,
&lengths_i32,
&lengths_array,
batch_size,
current_frames,
+ self.vocoder_data_type,
&kernels,
)?;
- // The final output is returned and held by the caller, so allocate a
- // fresh array instead of using a scratch buffer.
- x = tanh_enqueue(
+ let tanh_output = ws.next_scratch(
&mut encoder,
- &x,
- context.create_array_zeros(x.shape(), data_type, "structured_audio_tanh_output"),
- &kernels,
- )?;
+ &[batch_size, vocoder_graph.final_conv.cout, current_frames],
+ self.vocoder_data_type,
+ );
+ x = tanh_enqueue(&mut encoder, &x, tanh_output, &kernels)?;
let (completion_sender, completion_notification) = channel();
encoder.add_completion_handler({
@@ -376,7 +365,6 @@ impl StructuredAudioCodecGraph {
let _ = completion_sender.send(());
}
});
- let final_command_buffer = Some(encoder.end_encoding().submit());
let out_lengths = lengths_i32
.into_iter()
.map(|length| {
@@ -385,13 +373,16 @@ impl StructuredAudioCodecGraph {
})
})
.collect::<AudioResult<Vec<_>>>()?;
- Ok(SubmittedDecodedPaddedAudio {
- output: x,
+ let final_command_buffer = encoder.end_encoding().submit();
+ Ok(SubmittedDecodedPaddedAudio::Pending {
+ output: x.into_allocation(),
+ retained_inputs,
+ data_type: self.vocoder_data_type,
channels: vocoder_graph.final_conv.cout,
frames: current_frames,
lengths: out_lengths,
final_command_buffer,
- completion_notification: Some(completion_notification),
+ completion_notification,
})
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
index 5f0051723..827917577 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
@@ -1,5 +1,5 @@
use super::*;
-use crate::backends::common::gpu_types::ActivationType;
+use crate::{array::Array, backends::common::gpu_types::ActivationType};
pub(super) fn snake1d_enqueue<B: Backend>(
encoder: &mut Encoder<B>,
@@ -9,44 +9,33 @@ pub(super) fn snake1d_enqueue<B: Backend>(
batch_size: usize,
channels: usize,
seq_len: usize,
+ data_type: DataType,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
- let expected_input = checked_product(&[batch_size, channels, seq_len])?;
- if input.num_elements() != expected_input {
+ let expected_input = size_for_shape(&[batch_size, channels, seq_len], data_type);
+ if input.size() != expected_input {
return Err(AudioError::InvalidTokenShape {
expected_tokens: expected_input,
- actual_tokens: input.num_elements(),
+ actual_tokens: input.size(),
});
}
- if alpha.shape() != [channels] {
+ if alpha.size() != size_for_shape(&[channels], data_type) {
return Err(AudioError::InvalidTokenShape {
expected_tokens: channels,
actual_tokens: alpha.num_elements(),
});
}
- let data_type = input.data_type();
- if alpha.data_type() != data_type {
- return Err(AudioError::Runtime(format!(
- "snake alpha dtype mismatch: expected {:?}, got {:?}",
- data_type,
- alpha.data_type()
- )));
- }
-
let channels_i32 = usize_to_i32(channels, "channels")?;
let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let alpha_buffer = alpha.buffer();
- let alpha_buffer = alpha_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
kernels.half_snake.encode(
- &*input_buffer,
- &*alpha_buffer,
- &mut *output_buffer,
+ input.allocation(),
+ alpha.allocation(),
+ &mut output,
channels_i32,
seq_len_i32,
channels_i32,
@@ -56,11 +45,12 @@ pub(super) fn snake1d_enqueue<B: Backend>(
encoder,
);
- Ok(output)
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
fn validate_grouped_conv_params<B: Backend>(
input: &Array<B>,
+ data_type: DataType,
layer: &StructuredAudioConv1d<B>,
lengths: &[i32],
batch_size: usize,
@@ -76,31 +66,22 @@ fn validate_grouped_conv_params<B: Backend>(
return Err(AudioError::InvalidTokenCardinality);
}
let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];
- if layer.weight.shape() != expected_weight_shape {
+ if layer.weight.size() != size_for_shape(&expected_weight_shape, data_type) {
return Err(AudioError::Runtime(format!(
- "causal conv1d weight shape mismatch: expected {:?}, got {:?}",
- expected_weight_shape,
- layer.weight.shape()
+ "causal conv1d weight shape mismatch: expected {:?}",
+ expected_weight_shape
)));
}
- if layer.bias.shape() != [layer.cout] {
- return Err(AudioError::Runtime(format!(
- "causal conv1d bias shape mismatch: expected [{}], got {:?}",
- layer.cout,
- layer.bias.shape()
- )));
+ if layer.bias.size() != size_for_shape(&[layer.cout], data_type) {
+ return Err(AudioError::Runtime(format!("causal conv1d bias shape mismatch: expected [{}]", layer.cout)));
}
- let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;
- if input.num_elements() != expected_input {
+ let expected_input = size_for_shape(&[batch_size, layer.cin, seq_len], data_type);
+ if input.size() != expected_input {
return Err(AudioError::InvalidTokenShape {
expected_tokens: expected_input,
- actual_tokens: input.num_elements(),
+ actual_tokens: input.size(),
});
}
- let data_type = input.data_type();
- if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {
- return Err(AudioError::Runtime("causal conv1d dtype mismatch".to_string()));
- }
Ok(())
}
@@ -114,9 +95,10 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(
lengths_array: &Array<B>,
batch_size: usize,
seq_len: usize,
+ data_type: DataType,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
- validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;
+ validate_grouped_conv_params(input, data_type, layer, lengths, batch_size, seq_len)?;
let cin_i32 = usize_to_i32(layer.cin, "cin")?;
let cout_i32 = usize_to_i32(layer.cout, "cout")?;
@@ -125,23 +107,16 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(
let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;
let input_layout_i32 = input_layout.as_i32();
let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
if layer.groups == 1 {
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let weight_buffer = layer.weight.buffer();
- let weight_buffer = weight_buffer.borrow();
- let bias_buffer = layer.bias.buffer();
- let bias_buffer = bias_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
kernels.causal_conv1d.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input.allocation(),
+ layer.weight.allocation(),
+ layer.bias.allocation(),
+ &mut output,
+ lengths_array.allocation(),
cin_i32,
cout_i32,
seq_len_i32,
@@ -153,22 +128,12 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(
);
} else {
let groups_i32 = usize_to_i32(layer.groups, "groups")?;
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let weight_buffer = layer.weight.buffer();
- let weight_buffer = weight_buffer.borrow();
- let bias_buffer = layer.bias.buffer();
- let bias_buffer = bias_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
kernels.causal_conv1d_grouped.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input.allocation(),
+ layer.weight.allocation(),
+ layer.bias.allocation(),
+ &mut output,
+ lengths_array.allocation(),
cin_i32,
cout_i32,
seq_len_i32,
@@ -181,7 +146,7 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(
);
}
- Ok(output)
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(
@@ -194,14 +159,15 @@ pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(
lengths_array: &Array<B>,
batch_size: usize,
seq_len: usize,
+ data_type: DataType,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
- validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;
- let expected_residual = checked_product(&[batch_size, layer.cout, seq_len])?;
- if residual.num_elements() != expected_residual {
+ validate_grouped_conv_params(input, data_type, layer, lengths, batch_size, seq_len)?;
+ let expected_residual = size_for_shape(&[batch_size, layer.cout, seq_len], data_type);
+ if residual.size() != expected_residual {
return Err(AudioError::InvalidTokenShape {
expected_tokens: expected_residual,
- actual_tokens: residual.num_elements(),
+ actual_tokens: residual.size(),
});
}
let cin_i32 = usize_to_i32(layer.cin, "cin")?;
@@ -211,30 +177,16 @@ pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(
let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;
let groups_i32 = usize_to_i32(layer.groups, "groups")?;
let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
- let data_type = input.data_type();
- if residual.data_type() != data_type {
- return Err(AudioError::Runtime("causal conv1d residual dtype mismatch".to_string()));
- }
-
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let residual_buffer = residual.buffer();
- let residual_buffer = residual_buffer.borrow();
- let weight_buffer = layer.weight.buffer();
- let weight_buffer = weight_buffer.borrow();
- let bias_buffer = layer.bias.buffer();
- let bias_buffer = bias_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
kernels.causal_conv1d_grouped_residual.encode(
- &*input_buffer,
- &*residual_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input.allocation(),
+ residual.allocation(),
+ layer.weight.allocation(),
+ layer.bias.allocation(),
+ &mut output,
+ lengths_array.allocation(),
cin_i32,
cout_i32,
seq_len_i32,
@@ -245,7 +197,7 @@ pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(
encoder,
);
- Ok(output)
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(
@@ -259,6 +211,7 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(
seq_len_out: usize,
input_layout: SequenceLayout,
lengths_array: &Array<B>,
+ data_type: DataType,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
if lengths.len() != batch_size {
@@ -267,35 +220,25 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(
actual_lengths: lengths.len(),
});
}
- let expected_input = checked_product(&[batch_size, layer.cin, seq_len_in])?;
- if input.num_elements() != expected_input {
+ let expected_input = size_for_shape(&[batch_size, layer.cin, seq_len_in], data_type);
+ if input.size() != expected_input {
return Err(AudioError::InvalidTokenShape {
expected_tokens: expected_input,
- actual_tokens: input.num_elements(),
+ actual_tokens: input.size(),
});
}
if layer.groups == 0 || layer.cin % layer.groups != 0 || layer.cout % layer.groups != 0 {
return Err(AudioError::InvalidTokenCardinality);
}
let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];
- if layer.weight.shape() != expected_weight_shape {
+ if layer.weight.size() != size_for_shape(&expected_weight_shape, data_type) {
return Err(AudioError::Runtime(format!(
- "causal transpose weight shape mismatch: expected {:?}, got {:?}",
- expected_weight_shape,
- layer.weight.shape()
+ "causal transpose weight shape mismatch: expected {:?}",
+ expected_weight_shape
)));
}
- if layer.bias.shape() != [layer.cout] {
- return Err(AudioError::Runtime(format!(
- "causal transpose bias shape mismatch: expected [{}], got {:?}",
- layer.cout,
- layer.bias.shape()
- )));
- }
-
- let data_type = input.data_type();
- if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {
- return Err(AudioError::Runtime("causal transpose dtype mismatch".to_string()));
+ if layer.bias.size() != size_for_shape(&[layer.cout], data_type) {
+ return Err(AudioError::Runtime(format!("causal transpose bias shape mismatch: expected [{}]", layer.cout)));
}
let cin_i32 = usize_to_i32(layer.cin, "cin")?;
@@ -307,23 +250,15 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(
let groups_i32 = usize_to_i32(layer.groups, "groups")?;
let input_layout_i32 = input_layout.as_i32();
let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
-
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let weight_buffer = layer.weight.buffer();
- let weight_buffer = weight_buffer.borrow();
- let bias_buffer = layer.bias.buffer();
- let bias_buffer = bias_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
kernels.causal_conv_transpose1d_causal_pad.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input.allocation(),
+ layer.weight.allocation(),
+ layer.bias.allocation(),
+ &mut output,
+ lengths_array.allocation(),
cin_i32,
cout_i32,
seq_in_i32,
@@ -336,7 +271,7 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(
encoder,
);
- Ok(output)
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(
@@ -348,6 +283,7 @@ pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(
lengths_array: &Array<B>,
batch_size: usize,
seq_len: usize,
+ data_type: DataType,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
if lengths.len() != batch_size {
@@ -356,55 +292,37 @@ pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(
actual_lengths: lengths.len(),
});
}
- let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;
- if input.num_elements() != expected_input {
- return Err(AudioError::Runtime(format!(
- "pointwise conv input shape mismatch: expected {expected_input} elements ([batch={batch_size}, cin={}, seq_len={seq_len}]), got {}",
- layer.cin,
- input.num_elements()
- )));
- }
- if layer.weight.shape() != [layer.cout, layer.cin, 1] {
+ let expected_input = size_for_shape(&[batch_size, layer.cin, seq_len], data_type);
+ if input.size() != expected_input {
return Err(AudioError::Runtime(format!(
- "pointwise conv weight shape mismatch: expected [{}, {}, 1], got {:?}",
- layer.cout,
+ "pointwise conv input shape mismatch: expected {expected_input} bytes ([batch={batch_size}, cin={}, seq_len={seq_len}]), got {}",
layer.cin,
- layer.weight.shape()
+ input.size()
)));
}
- if layer.bias.shape() != [layer.cout] {
+ if layer.weight.size() != size_for_shape(&[layer.cout, layer.cin], data_type) {
return Err(AudioError::Runtime(format!(
- "pointwise conv bias shape mismatch: expected [{}], got {:?}",
- layer.cout,
- layer.bias.shape()
+ "pointwise conv weight shape mismatch: expected [{}, {}]",
+ layer.cout, layer.cin
)));
}
-
- let data_type = input.data_type();
- if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {
- return Err(AudioError::Runtime("pointwise conv dtype mismatch".to_string()));
+ if layer.bias.size() != size_for_shape(&[layer.cout], data_type) {
+ return Err(AudioError::Runtime(format!("pointwise conv bias shape mismatch: expected [{}]", layer.cout)));
}
let cin_i32 = usize_to_i32(layer.cin, "cin")?;
let cout_i32 = usize_to_i32(layer.cout, "cout")?;
let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let weight_buffer = layer.weight.buffer();
- let weight_buffer = weight_buffer.borrow();
- let bias_buffer = layer.bias.buffer();
- let bias_buffer = bias_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
kernels.conv1d.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input.allocation(),
+ layer.weight.allocation(),
+ layer.bias.allocation(),
+ &mut output,
+ lengths_array.allocation(),
cin_i32,
cout_i32,
seq_len_i32,
@@ -418,7 +336,7 @@ pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(
encoder,
);
- Ok(output)
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
pub(super) fn norm_ncs_enqueue<B: Backend>(
@@ -431,6 +349,7 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(
batch_size: usize,
channels: usize,
seq_len: usize,
+ data_type: DataType,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
if lengths.len() != batch_size {
@@ -439,32 +358,21 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(
actual_lengths: lengths.len(),
});
}
- if norm.scales.shape() != [channels] {
- return Err(AudioError::Runtime(format!(
- "norm scale shape mismatch: expected [{channels}], got {:?}",
- norm.scales.shape()
- )));
+ if norm.scales.size() != size_for_shape(&[channels], data_type) {
+ return Err(AudioError::Runtime(format!("norm scale shape mismatch: expected [{channels}]")));
}
- if norm.bias.shape() != [channels] {
- return Err(AudioError::Runtime(format!(
- "norm bias shape mismatch: expected [{channels}], got {:?}",
- norm.bias.shape()
- )));
+ if norm.bias.size() != size_for_shape(&[channels], data_type) {
+ return Err(AudioError::Runtime(format!("norm bias shape mismatch: expected [{channels}]")));
}
- let expected_input = checked_product(&[batch_size, channels, seq_len])?;
- if input.num_elements() != expected_input {
+ let expected_input = size_for_shape(&[batch_size, channels, seq_len], data_type);
+ if input.size() != expected_input {
return Err(AudioError::InvalidTokenShape {
expected_tokens: expected_input,
- actual_tokens: input.num_elements(),
+ actual_tokens: input.size(),
});
}
- let data_type = input.data_type();
- if norm.scales.data_type() != data_type || norm.bias.data_type() != data_type {
- return Err(AudioError::Runtime("norm dtype mismatch".to_string()));
- }
-
let channels_i32 = usize_to_i32(channels, "channels")?;
let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
@@ -473,22 +381,15 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(
} else {
0
};
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let scales_buffer = norm.scales.buffer();
- let scales_buffer = scales_buffer.borrow();
- let bias_buffer = norm.bias.buffer();
- let bias_buffer = bias_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- let lengths_buffer = lengths_array.buffer();
- let lengths_buffer = lengths_buffer.borrow();
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
kernels.norm_ncs.encode(
- &*input_buffer,
- &*scales_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input.allocation(),
+ norm.scales.allocation(),
+ norm.bias.allocation(),
+ &mut output,
+ lengths_array.allocation(),
channels_i32,
seq_len_i32,
norm.epsilon,
@@ -497,7 +398,7 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(
encoder,
);
- Ok(output)
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
pub(super) fn gelu_enqueue<B: Backend>(
@@ -508,12 +409,11 @@ pub(super) fn gelu_enqueue<B: Backend>(
) -> AudioResult<Array<B>> {
let n_u32 = u32::try_from(input.num_elements())
.map_err(|_| AudioError::Runtime("gelu element count exceeds u32 range".to_string()))?;
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::GELU, encoder);
- Ok(output)
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
+ kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, ActivationType::GELU, encoder);
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
pub(super) fn add_enqueue<B: Backend>(
@@ -523,29 +423,15 @@ pub(super) fn add_enqueue<B: Backend>(
output: Array<B>,
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
- if a.num_elements() != b.num_elements() {
- return Err(AudioError::Runtime(format!(
- "elementwise add shape mismatch: {} vs {}",
- a.num_elements(),
- b.num_elements()
- )));
- }
- if a.data_type() != b.data_type() {
- return Err(AudioError::Runtime(format!(
- "elementwise add dtype mismatch: {:?} vs {:?}",
- a.data_type(),
- b.data_type()
- )));
+ if a.size() != b.size() {
+ return Err(AudioError::Runtime(format!("elementwise add shape mismatch: {} vs {}", a.size(), b.size())));
}
let n_i32 = usize_to_i32(a.num_elements(), "n")?;
- let a_buffer = a.buffer();
- let a_buffer = a_buffer.borrow();
- let b_buffer = b.buffer();
- let b_buffer = b_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- kernels.add.encode(&*a_buffer, &*b_buffer, &mut *output_buffer, n_i32, encoder);
- Ok(output)
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
+ kernels.add.encode(a.allocation(), b.allocation(), &mut output, n_i32, encoder);
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
pub(super) fn tanh_enqueue<B: Backend>(
@@ -556,10 +442,9 @@ pub(super) fn tanh_enqueue<B: Backend>(
) -> AudioResult<Array<B>> {
let n_u32 = u32::try_from(input.num_elements())
.map_err(|_| AudioError::Runtime("tanh element count exceeds u32 range".to_string()))?;
- let input_buffer = input.buffer();
- let input_buffer = input_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::TANH, encoder);
- Ok(output)
+ let output_shape = output.shape().to_vec();
+ let output_data_type = output.data_type();
+ let mut output = output.into_allocation();
+ kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, ActivationType::TANH, encoder);
+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
index 1ef48c5d3..df09c9349 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
@@ -1,5 +1,5 @@
use super::*;
-use crate::backends::common::Buffer;
+use crate::array::{Array, ArrayContextExt};
pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {
match data_type {
@@ -12,8 +12,7 @@ pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {
pub(super) struct StructuredAudioPostModuleRuntime<B: Backend> {
pub(super) context: Rc<B::Context>,
pub(super) model_shape: ModelShape,
- pub(super) scratch_buffers: ScratchBuffers<B>,
- pub(super) shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
+ pub(super) shared_buffers: Rc<SharedBuffers<B>>,
pub(super) layers: Box<[LayerExecutables<B>]>,
pub(super) output_norm: RMSNorm<B>,
pub(super) max_sequence_length: usize,
@@ -46,134 +45,46 @@ pub(super) struct FishAudioQuantizerResources<B: Backend> {
pub(super) residual_out_bias: Array<B>,
}
-// TODO: once `Array<B>` is bridged to `backends::common::allocator::Allocation`
-// project-wide, replace this whole workspace with `Encoder::allocate_scratch`.
-// That pool already handles auto-expand on demand, range aliasing for
-// non-overlapping lifetimes, and free-on-drop tied to command-buffer
-// completion — i.e. everything this struct reimplements by hand.
pub(super) struct DecodeWorkspace<B: Backend> {
- slots: RefCell<Vec<Rc<RefCell<B::DenseBuffer>>>>,
- next_search: Cell<usize>,
- min_buffer_size: Cell<usize>,
- lengths_pair: [RefCell<Option<Array<B>>>; 2],
+ _marker: std::marker::PhantomData<B>,
}
impl<B: Backend> DecodeWorkspace<B> {
fn new() -> Self {
Self {
- slots: RefCell::new(Vec::new()),
- next_search: Cell::new(0),
- min_buffer_size: Cell::new(0),
- lengths_pair: [RefCell::new(None), RefCell::new(None)],
+ _marker: std::marker::PhantomData,
}
}
- pub(super) fn prime_for(
- &self,
- context: &B::Context,
- bytes: usize,
- ) -> AudioResult<()> {
- if bytes <= self.min_buffer_size.get() {
- return Ok(());
- }
- self.min_buffer_size.set(bytes);
- let mut slots = self.slots.borrow_mut();
- for slot in slots.iter_mut() {
- if Rc::strong_count(slot) == 1 && slot.borrow().size() < bytes {
- *slot = create_labelled_buffer::<B>(context, bytes, "structured_audio_scratch")?;
- }
- }
- Ok(())
- }
-
pub(super) fn lengths_array(
&self,
- context: &B::Context,
- index: usize,
+ encoder: &mut Encoder<B>,
min_elements: usize,
) -> Array<B> {
- debug_assert!(index < 2);
- let mut slot = self.lengths_pair[index].borrow_mut();
- if let Some(existing) = slot.as_ref() {
- if existing.num_elements() >= min_elements {
- return existing.clone();
- }
- }
- let label = if index == 0 {
- "structured_audio_lengths_a"
- } else {
- "structured_audio_lengths_b"
- };
- let array = context.create_array_zeros(&[min_elements], DataType::I32, label);
- *slot = Some(array.clone());
- array
+ let allocation = encoder
+ .allocate_constant(min_elements * std::mem::size_of::<i32>())
+ .expect("Failed to allocate structured audio lengths");
+ unsafe { Array::from_allocation(allocation, 0, &[min_elements], DataType::I32) }
}
pub(super) fn next_scratch(
&self,
- context: &B::Context,
+ encoder: &mut Encoder<B>,
shape: &[usize],
data_type: DataType,
- label: &str,
- ) -> AudioResult<Array<B>> {
- let needed = size_for_shape(shape, data_type).max(self.min_buffer_size.get());
- let mut slots = self.slots.borrow_mut();
- let slot_count = slots.len();
-
- if slot_count > 0 {
- let start = self.next_search.get() % slot_count;
- for offset in 0..slot_count {
- let slot_index = (start + offset) % slot_count;
- if Rc::strong_count(&slots[slot_index]) == 1 && slots[slot_index].borrow().size() >= needed {
- self.next_search.set((slot_index + 1) % slot_count);
- return Ok(unsafe { Array::from_parts(slots[slot_index].clone(), 0, shape, data_type) });
- }
- }
- for offset in 0..slot_count {
- let slot_index = (start + offset) % slot_count;
- if Rc::strong_count(&slots[slot_index]) == 1 {
- slots[slot_index] = create_labelled_buffer::<B>(context, needed, label)?;
- self.next_search.set((slot_index + 1) % slot_count);
- return Ok(unsafe { Array::from_parts(slots[slot_index].clone(), 0, shape, data_type) });
- }
- }
- }
-
- let buffer_rc = create_labelled_buffer::<B>(context, needed, label)?;
- slots.push(buffer_rc.clone());
- self.next_search.set(0);
- Ok(unsafe { Array::from_parts(buffer_rc, 0, shape, data_type) })
- }
-
- pub(super) fn reset(&self) {
- self.slots.borrow_mut().clear();
- self.next_search.set(0);
- self.min_buffer_size.set(0);
- *self.lengths_pair[0].borrow_mut() = None;
- *self.lengths_pair[1].borrow_mut() = None;
+ ) -> Array<B> {
+ let allocation =
+ encoder.allocate_scratch(size_for_shape(shape, data_type)).expect("Failed to allocate scratch");
+ unsafe { Array::from_allocation(allocation, 0, shape, data_type) }
}
}
-fn create_labelled_buffer<B: Backend>(
- context: &B::Context,
- bytes: usize,
- label: &str,
-) -> AudioResult<Rc<RefCell<B::DenseBuffer>>> {
- let mut buffer = context
- .create_buffer(bytes)
- .map_err(|err| AudioError::Runtime(format!("failed to create scratch buffer '{label}': {err}")))?;
- buffer.set_label(Some(label));
- Ok(Rc::new(RefCell::new(buffer)))
-}
-
pub(in crate::audio::nanocodec::runtime) struct StructuredAudioRuntimeResources<B: Backend> {
context: Rc<B::Context>,
pub(super) kernels_by_dtype: RefCell<HashMap<u8, Rc<StructuredAudioKernelCache<B>>>>,
pub(super) post_module_runtime: RefCell<Option<Rc<StructuredAudioPostModuleRuntime<B>>>>,
pub(super) vocoder_graph: RefCell<Option<Rc<StructuredAudioDecoderGraph<B>>>>,
pub(super) quantizer_resources: RefCell<Option<Rc<FishAudioQuantizerResources<B>>>>,
- token_staging: RefCell<Option<Array<B>>>,
- length_staging: RefCell<Option<Array<B>>>,
pub(super) decode_workspace: DecodeWorkspace<B>,
}
@@ -185,8 +96,6 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {
post_module_runtime: RefCell::new(None),
vocoder_graph: RefCell::new(None),
quantizer_resources: RefCell::new(None),
- token_staging: RefCell::new(None),
- length_staging: RefCell::new(None),
decode_workspace: DecodeWorkspace::new(),
}
}
@@ -195,15 +104,6 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {
&self.context
}
- /// Reset the decode workspace and staging arrays so that subsequent
- /// decode passes allocate fresh buffers. Call this after submitting a
- /// command buffer whose encoded work references the current buffers.
- pub(in crate::audio::nanocodec::runtime) fn reset_for_pending(&self) {
- self.decode_workspace.reset();
- *self.token_staging.borrow_mut() = None;
- *self.length_staging.borrow_mut() = None;
- }
-
pub(super) fn kernels(
&self,
data_type: DataType,
@@ -221,30 +121,14 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {
&self,
min_elements: usize,
) -> Array<B> {
- let mut slot = self.token_staging.borrow_mut();
- if let Some(existing) = slot.as_ref() {
- if existing.num_elements() >= min_elements {
- return existing.clone();
- }
- }
- let array = self.context.create_array_zeros(&[min_elements], DataType::U32, "quantizer_token_staging");
- *slot = Some(array.clone());
- array
+ self.context.create_array_uninitialized(&[min_elements], DataType::U32)
}
pub(super) fn length_staging(
&self,
min_elements: usize,
) -> Array<B> {
- let mut slot = self.length_staging.borrow_mut();
- if let Some(existing) = slot.as_ref() {
- if existing.num_elements() >= min_elements {
- return existing.clone();
- }
- }
- let array = self.context.create_array_zeros(&[min_elements], DataType::I32, "quantizer_length_staging");
- *slot = Some(array.clone());
- array
+ self.context.create_array_uninitialized(&[min_elements], DataType::I32)
}
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 53f5fdf31..4b871eabd 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -1,4 +1,10 @@
use super::*;
+use crate::{
+ array::{Array, ArrayContextExt},
+ backends::common::{Allocation, AsBufferRangeRef},
+ encodable_block::LayerArguments,
+ forward_pass::token_inputs::TokenInputs,
+};
impl StructuredAudioCodecGraph {
pub(super) fn apply_convnext_ncs_enqueued<B: Backend>(
@@ -15,58 +21,65 @@ impl StructuredAudioCodecGraph {
kernels: &StructuredAudioKernelCache<B>,
) -> AudioResult<Array<B>> {
let ws = &resources.decode_workspace;
- let ctx = resources.context();
- let data_type = input.data_type();
- let residual = input.clone();
- let mut x = causal_conv1d_grouped_enqueue(
+ let cnxt_dw = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);
+ let x = causal_conv1d_grouped_enqueue(
encoder,
input,
- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_dw")?,
+ cnxt_dw,
&layer.depthwise_conv,
SequenceLayout::Ncs,
lengths,
lengths_array,
batch_size,
seq_len,
+ self.vocoder_data_type,
kernels,
)?;
- x = norm_ncs_enqueue(
+ let cnxt_norm = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);
+ let x = norm_ncs_enqueue(
encoder,
&x,
- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_norm")?,
+ cnxt_norm,
&layer.norm,
lengths,
lengths_array,
batch_size,
channels,
seq_len,
+ self.vocoder_data_type,
kernels,
)?;
- x = conv1d_pointwise_ncs_enqueue(
+ let cnxt_pw1 = ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);
+ let x = conv1d_pointwise_ncs_enqueue(
encoder,
&x,
- ws.next_scratch(ctx, &[batch_size, layer.pwconv1.cout, seq_len], data_type, "cnxt_pw1")?,
+ cnxt_pw1,
&layer.pwconv1,
lengths,
lengths_array,
batch_size,
seq_len,
+ self.vocoder_data_type,
kernels,
)?;
- x = gelu_enqueue(encoder, &x, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_gelu")?, kernels)?;
- x = conv1d_pointwise_ncs_enqueue(
+ let cnxt_gelu = ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);
+ let x = gelu_enqueue(encoder, &x, cnxt_gelu, kernels)?;
+ let cnxt_pw2 = ws.next_scratch(encoder, &[batch_size, layer.pwconv2.cout, seq_len], self.vocoder_data_type);
+ let x = conv1d_pointwise_ncs_enqueue(
encoder,
&x,
- ws.next_scratch(ctx, &[batch_size, layer.pwconv2.cout, seq_len], data_type, "cnxt_pw2")?,
+ cnxt_pw2,
&layer.pwconv2,
lengths,
lengths_array,
batch_size,
seq_len,
+ self.vocoder_data_type,
kernels,
)?;
- add_enqueue(encoder, &x, &residual, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_add")?, kernels)
+ let cnxt_add = ws.next_scratch(encoder, &[batch_size, layer.pwconv2.cout, seq_len], self.vocoder_data_type);
+ add_enqueue(encoder, &x, input, cnxt_add, kernels)
}
pub(super) fn build_post_module_runtime<B: Backend>(
@@ -103,12 +116,11 @@ impl StructuredAudioCodecGraph {
.map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
let max_sequence_length = decoder_config.context_length.max(required_sequence_length.max(1));
- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
{
let transformer_tree = root_loader_view
.subtree(transformer_subtree_name)
.map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
- let mut shared_buffers = shared_buffers.borrow_mut();
if let Some(global_rope) = &mut shared_buffers.global_rope {
global_rope.update_data(&transformer_tree, "global_rope");
}
@@ -116,7 +128,7 @@ impl StructuredAudioCodecGraph {
local_rope.update_data(&transformer_tree, "local_rope");
}
}
- let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_sequence_length);
+ let shared_buffers = Rc::new(shared_buffers);
let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(
context.as_ref(),
&decoder_config,
@@ -127,7 +139,6 @@ impl StructuredAudioCodecGraph {
Ok(StructuredAudioPostModuleRuntime {
context,
model_shape,
- scratch_buffers,
shared_buffers,
layers,
output_norm,
@@ -180,7 +191,7 @@ impl StructuredAudioCodecGraph {
self.config.codebook_dim,
data_type,
)?;
- let codebook_dim = semantic_codebook.shape()[1];
+ let codebook_dim = self.config.codebook_dim;
let semantic_out_proj = read_float_matrix_exact::<B>(
&semantic_tree.subtree("out_proj")?,
"weights",
@@ -194,21 +205,11 @@ impl StructuredAudioCodecGraph {
let residual_quantizers = self.config.n_codebooks;
let residual_count_for_shape = residual_quantizers.max(1);
let residual_codebook_rows_for_shape = self.codebook_size.max(1);
- let residual_codebooks = context.create_array_zeros(
- &[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim],
- data_type,
- "structured_audio_quantizer_residual_codebooks",
- );
- let residual_out_proj = context.create_array_zeros(
- &[residual_count_for_shape, self.input_dim, codebook_dim],
- data_type,
- "structured_audio_quantizer_residual_out_proj",
- );
- let residual_out_bias = context.create_array_zeros(
- &[residual_count_for_shape, self.input_dim],
- data_type,
- "structured_audio_quantizer_residual_out_bias",
- );
+ let mut residual_codebooks = context
+ .create_array_zeros(&[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim], data_type);
+ let mut residual_out_proj =
+ context.create_array_zeros(&[residual_count_for_shape, self.input_dim, codebook_dim], data_type);
+ let mut residual_out_bias = context.create_array_zeros(&[residual_count_for_shape, self.input_dim], data_type);
let residual_root = quantizer_tree.subtree("quantizer")?.subtree("quantizers")?;
for index in 0..residual_quantizers {
@@ -234,12 +235,9 @@ impl StructuredAudioCodecGraph {
data_type,
)?;
- let mut dst_codebook = outer_axis_view(&residual_codebooks, index, &[self.codebook_size, codebook_dim])?;
- dst_codebook.copy_from_array(&codebook);
- let mut dst_out_proj = outer_axis_view(&residual_out_proj, index, &[self.input_dim, codebook_dim])?;
- dst_out_proj.copy_from_array(&out_proj);
- let mut dst_out_bias = outer_axis_view(&residual_out_bias, index, &[self.input_dim])?;
- dst_out_bias.copy_from_array(&out_bias);
+ copy_to_outer_axis_slice(&mut residual_codebooks, index, &codebook, &[self.codebook_size, codebook_dim])?;
+ copy_to_outer_axis_slice(&mut residual_out_proj, index, &out_proj, &[self.input_dim, codebook_dim])?;
+ copy_to_outer_axis_slice(&mut residual_out_bias, index, &out_bias, &[self.input_dim])?;
}
Ok(FishAudioQuantizerResources {
@@ -298,100 +296,82 @@ impl StructuredAudioCodecGraph {
pub(super) fn encode_post_module_layers<B: Backend>(
runtime: &StructuredAudioPostModuleRuntime<B>,
- state: &mut ForwardPassState<B>,
+ token_inputs: &TokenInputs<B>,
+ batch_dim: usize,
+ main: Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> AudioResult<()> {
+ ) -> AudioResult<Allocation<B>> {
let encoding_parameters = EncodingParameters::new();
+ let mut main = main;
+ let mut shortcut = encoder
+ .allocate_scratch(main.as_buffer_range_ref().range().len())
+ .map_err(|err| AudioError::Runtime(format!("post_module shortcut allocation failed: {err}")))?;
for layer in runtime.layers.iter() {
- layer
- .encode(state, &encoding_parameters, encoder)
+ let rope_type = layer.rope_type();
+ main = layer
+ .encode(
+ LayerArguments {
+ batch_dim,
+ token_positions: token_inputs.token_positions(),
+ token_parents: token_inputs.token_parents(),
+ token_subtrie_ranges: None,
+ attention_sinks: runtime.shared_buffers.attention_sinks(layer.layer_index),
+ rope_cosines: rope_type.and_then(|rope_type| runtime.shared_buffers.rope_cosines(rope_type)),
+ rope_sines: rope_type.and_then(|rope_type| runtime.shared_buffers.rope_sines(rope_type)),
+ rope_max_sequence_length: runtime.model_shape.context_length(),
+ rope_dim: runtime.model_shape.rope_dim(),
+ sampling_start: 0,
+ sampling_length: batch_dim,
+ cache_layer: None,
+ #[cfg(feature = "tracing")]
+ trace: None,
+ },
+ &encoding_parameters,
+ main,
+ &mut shortcut,
+ encoder,
+ )
.map_err(|err| AudioError::Runtime(format!("post_module layer encode failed: {err}")))?;
}
runtime
.output_norm
- .encode(state, encoder)
- .map_err(|err| AudioError::Runtime(format!("post_module output norm encode failed: {err}")))?;
- Ok(())
+ .encode(&main, 0, batch_dim, Some(&mut shortcut), encoder)
+ .map_err(|err| AudioError::Runtime(format!("post_module output norm encode failed: {err}")))
}
pub(super) fn apply_post_module_single_batch_enqueued<B: Backend>(
&self,
resources: &StructuredAudioRuntimeResources<B>,
encoder: &mut Encoder<B>,
- latent_nsc: &Array<B>,
+ latent_nsc: Array<B>,
frames: usize,
) -> AudioResult<Array<B>> {
if frames == 0 {
- return Ok(latent_nsc.clone());
+ return Ok(latent_nsc);
}
- let expected_elements = checked_product(&[frames, self.input_dim])?;
- if latent_nsc.num_elements() != expected_elements {
+ let expected_size = size_for_shape(&[frames, self.input_dim], self.vocoder_data_type);
+ if latent_nsc.size() != expected_size {
return Err(AudioError::InvalidTokenShape {
- expected_tokens: expected_elements,
- actual_tokens: latent_nsc.num_elements(),
+ expected_tokens: expected_size,
+ actual_tokens: latent_nsc.size(),
});
}
let runtime = self.post_module_runtime(resources, frames.max(1))?;
let token_ids = vec![0_u64; frames];
let token_positions = (0..frames).collect::<Vec<_>>();
- let mut state = ForwardPassState::new_classifier(
- runtime.context.clone(),
- &runtime.model_shape,
- &runtime.scratch_buffers,
- runtime.shared_buffers.clone(),
- &token_ids,
- &token_positions,
- 1,
- );
-
- let main = state.array(ArrayId::Main);
- let main_output = {
- if main.shape() != [frames, self.input_dim] {
- return Err(AudioError::Runtime(format!(
- "post_module main shape mismatch: expected [{frames}, {}], got {:?}",
- self.input_dim,
- main.shape()
- )));
- }
- if main.data_type() != latent_nsc.data_type() {
- return Err(AudioError::Runtime(format!(
- "post_module dtype mismatch: main={:?}, latent={:?}",
- main.data_type(),
- latent_nsc.data_type()
- )));
- }
- main.clone()
- };
-
- let copy_bytes = latent_nsc
- .num_elements()
- .checked_mul(latent_nsc.data_type().size_in_bytes())
- .ok_or(AudioError::Runtime("post_module copy size overflow".to_string()))?;
- {
- let latent_buffer = latent_nsc.buffer();
- let main_output_buffer = main_output.buffer();
- let latent_buffer = latent_buffer.borrow();
- let mut main_output_buffer = main_output_buffer.borrow_mut();
- encoder.encode_copy(
- &latent_buffer,
- latent_nsc.offset()..latent_nsc.offset() + copy_bytes,
- &mut main_output_buffer,
- main_output.offset()..main_output.offset() + copy_bytes,
- );
- }
- Self::encode_post_module_layers(&runtime, &mut state, encoder)?;
-
- Ok(main_output)
+ let token_inputs = TokenInputs::new_classifier(runtime.context.as_ref(), &token_ids, &token_positions);
+ let main =
+ Self::encode_post_module_layers(&runtime, &token_inputs, frames, latent_nsc.into_allocation(), encoder)?;
+ Ok(unsafe { Array::from_allocation(main, 0, &[frames, self.input_dim], self.vocoder_data_type) })
}
pub(super) fn apply_post_module_enqueued<B: Backend>(
&self,
resources: &StructuredAudioRuntimeResources<B>,
- context: &Rc<B::Context>,
encoder: &mut Encoder<B>,
- latent_nsc: &Array<B>,
+ latent_nsc: Array<B>,
lengths: &[usize],
batch_size: usize,
frames: usize,
@@ -402,22 +382,13 @@ impl StructuredAudioCodecGraph {
actual_lengths: lengths.len(),
});
}
- let expected_elements = checked_product(&[batch_size, frames, self.input_dim])?;
- if latent_nsc.num_elements() != expected_elements {
+ let expected_size = size_for_shape(&[batch_size, frames, self.input_dim], self.vocoder_data_type);
+ if latent_nsc.size() != expected_size {
return Err(AudioError::InvalidTokenShape {
- expected_tokens: expected_elements,
- actual_tokens: latent_nsc.num_elements(),
+ expected_tokens: expected_size,
+ actual_tokens: latent_nsc.size(),
});
}
- if batch_size == 1 && lengths.first().copied() == Some(frames) {
- return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);
- }
-
- let output = context.create_array_zeros(
- &[batch_size, frames, self.input_dim],
- latent_nsc.data_type(),
- "structured_audio_post_module_output_nsc",
- );
let mut batch_indices_by_length = BTreeMap::<usize, Vec<usize>>::new();
for (batch_index, &active_len) in lengths.iter().enumerate() {
if active_len == 0 {
@@ -431,94 +402,84 @@ impl StructuredAudioCodecGraph {
}
batch_indices_by_length.entry(active_len).or_default().push(batch_index);
}
+ if frames == 0 {
+ return Ok(latent_nsc);
+ }
+ if batch_size == 1 && lengths.first().copied() == Some(frames) {
+ return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);
+ }
+ let mut output = encoder.allocate_scratch(expected_size).map_err(|err| {
+ AudioError::Runtime(format!("failed to allocate structured audio post_module output: {err}"))
+ })?;
if batch_indices_by_length.is_empty() {
- let mut output = output;
- output.copy_from_array(latent_nsc);
- return Ok(output);
+ encoder.encode_copy(
+ latent_nsc.allocation(),
+ latent_nsc.offset()..latent_nsc.offset() + latent_nsc.size(),
+ &mut output,
+ 0..latent_nsc.size(),
+ );
+ return Ok(unsafe {
+ Array::from_allocation(output, 0, &[batch_size, frames, self.input_dim], self.vocoder_data_type)
+ });
}
let full_copy_bytes = latent_nsc.size();
+ let batch_stride_bytes = size_for_shape(&[frames, self.input_dim], self.vocoder_data_type);
let mut copied_output_prefix = false;
for (active_len, batch_indices) in batch_indices_by_length {
let runtime = self.post_module_runtime(resources, active_len.max(1))?;
let token_ids = vec![0_u64; active_len];
let token_positions = (0..active_len).collect::<Vec<_>>();
- let mut state = ForwardPassState::new_classifier(
- runtime.context.clone(),
- &runtime.model_shape,
- &runtime.scratch_buffers,
- runtime.shared_buffers.clone(),
- &token_ids,
- &token_positions,
- 1,
- );
+ let token_inputs = TokenInputs::new_classifier(runtime.context.as_ref(), &token_ids, &token_positions);
- let main = state.array(ArrayId::Main);
- let main_output = {
- if main.shape() != [active_len, self.input_dim] {
- return Err(AudioError::Runtime(format!(
- "post_module main shape mismatch: expected [{active_len}, {}], got {:?}",
- self.input_dim,
- main.shape()
- )));
- }
- if main.data_type() != latent_nsc.data_type() {
- return Err(AudioError::Runtime(format!(
- "post_module dtype mismatch: main={:?}, latent={:?}",
- main.data_type(),
- latent_nsc.data_type()
- )));
- }
- main.clone()
- };
+ let main_shape = runtime.model_shape.main_shape(active_len);
+ if main_shape != [active_len, self.input_dim] {
+ return Err(AudioError::Runtime(format!(
+ "post_module main shape mismatch: expected [{active_len}, {}], got [{}, {}]",
+ self.input_dim, main_shape[0], main_shape[1]
+ )));
+ }
+ if runtime.model_shape.activation_data_type() != self.vocoder_data_type {
+ return Err(AudioError::Runtime(format!(
+ "post_module dtype mismatch: main={:?}, latent={:?}",
+ runtime.model_shape.activation_data_type(),
+ self.vocoder_data_type
+ )));
+ }
for &batch_index in &batch_indices {
if !copied_output_prefix {
- {
- let latent_buffer = latent_nsc.buffer();
- let output_buffer = output.buffer();
- let latent_buffer = latent_buffer.borrow();
- let mut output_buffer = output_buffer.borrow_mut();
- encoder.encode_copy(
- &latent_buffer,
- latent_nsc.offset()..latent_nsc.offset() + full_copy_bytes,
- &mut output_buffer,
- output.offset()..output.offset() + full_copy_bytes,
- );
- }
- copied_output_prefix = true;
- }
- let source = array_batch_view(latent_nsc, batch_index, frames, self.input_dim, active_len)?;
- {
- let source_buffer = source.buffer();
- let main_output_buffer = main_output.buffer();
- let source_buffer = source_buffer.borrow();
- let mut main_output_buffer = main_output_buffer.borrow_mut();
- encoder.encode_copy(
- &source_buffer,
- source.offset()..source.offset() + source.size(),
- &mut main_output_buffer,
- main_output.offset()..main_output.offset() + source.size(),
- );
- }
- Self::encode_post_module_layers(&runtime, &mut state, encoder)?;
- let destination = array_batch_view(&output, batch_index, frames, self.input_dim, active_len)?;
- {
- let main_output_buffer = main_output.buffer();
- let destination_buffer = destination.buffer();
- let main_output_buffer = main_output_buffer.borrow();
- let mut destination_buffer = destination_buffer.borrow_mut();
encoder.encode_copy(
- &main_output_buffer,
- main_output.offset()..main_output.offset() + destination.size(),
- &mut destination_buffer,
- destination.offset()..destination.offset() + destination.size(),
+ latent_nsc.allocation(),
+ latent_nsc.offset()..latent_nsc.offset() + full_copy_bytes,
+ &mut output,
+ 0..full_copy_bytes,
);
+ copied_output_prefix = true;
}
+ let source_range =
+ array_batch_byte_range(&latent_nsc, batch_index, frames, self.input_dim, active_len)?;
+ let main_array = resources.decode_workspace.next_scratch(
+ encoder,
+ &[active_len, self.input_dim],
+ self.vocoder_data_type,
+ );
+ let mut main = main_array.into_allocation();
+ let source_size = source_range.end - source_range.start;
+ encoder.encode_copy(latent_nsc.allocation(), source_range, &mut main, 0..source_size);
+ let main = Self::encode_post_module_layers(&runtime, &token_inputs, active_len, main, encoder)?;
+ let destination_offset = batch_index * batch_stride_bytes;
+ let active_bytes = active_len * self.input_dim * self.vocoder_data_type.size_in_bytes();
+ encoder.encode_copy(
+ &main,
+ 0..active_bytes,
+ &mut output,
+ destination_offset..destination_offset + active_bytes,
+ );
}
}
- Ok(output)
+ Ok(unsafe { Array::from_allocation(output, 0, &[batch_size, frames, self.input_dim], self.vocoder_data_type) })
}
}
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs
index 249ee8573..5677cdaec 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs
@@ -1,21 +1,27 @@
+use std::ops::Range;
+
use super::*;
+use crate::array::Array;
-pub(super) fn array_batch_view<B: Backend>(
+pub(super) fn array_batch_byte_range<B: Backend>(
array: &Array<B>,
batch_index: usize,
frames: usize,
channels: usize,
active_frames: usize,
-) -> AudioResult<Array<B>> {
+) -> AudioResult<Range<usize>> {
let batch_stride_bytes = size_for_shape(&[frames, channels], array.data_type());
let batch_offset = batch_index
.checked_mul(batch_stride_bytes)
.and_then(|value| value.checked_add(array.offset()))
- .ok_or(AudioError::Runtime("array batch view offset overflow".to_string()))?;
+ .ok_or(AudioError::Runtime("array batch range offset overflow".to_string()))?;
if active_frames > frames {
- return Err(AudioError::Runtime("array batch view active_frames exceeds frames".to_string()));
+ return Err(AudioError::Runtime("array batch range active_frames exceeds frames".to_string()));
}
- Ok(unsafe { Array::from_parts(array.buffer(), batch_offset, &[active_frames, channels], array.data_type()) })
+ let byte_len = size_for_shape(&[active_frames, channels], array.data_type());
+ let end =
+ batch_offset.checked_add(byte_len).ok_or(AudioError::Runtime("array batch range end overflow".to_string()))?;
+ Ok(batch_offset..end)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -45,7 +51,7 @@ pub struct StructuredAudioCodecGraph {
pub(in crate::audio::nanocodec::runtime) vocoder_data_type: DataType,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioConv1d<B: Backend> {
pub(super) weight: Array<B>,
pub(super) bias: Array<B>,
@@ -56,7 +62,7 @@ pub(super) struct StructuredAudioConv1d<B: Backend> {
pub(super) groups: usize,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioConvTranspose1d<B: Backend> {
pub(super) weight: Array<B>,
pub(super) bias: Array<B>,
@@ -67,7 +73,7 @@ pub(super) struct StructuredAudioConvTranspose1d<B: Backend> {
pub(super) groups: usize,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioPointwiseConv<B: Backend> {
pub(super) weight: Array<B>,
pub(super) bias: Array<B>,
@@ -75,7 +81,7 @@ pub(super) struct StructuredAudioPointwiseConv<B: Backend> {
pub(super) cout: usize,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioNorm<B: Backend> {
pub(super) scales: Array<B>,
pub(super) bias: Array<B>,
@@ -83,7 +89,7 @@ pub(super) struct StructuredAudioNorm<B: Backend> {
pub(super) subtract_mean: bool,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioConvNeXt<B: Backend> {
pub(super) depthwise_conv: StructuredAudioConv1d<B>,
pub(super) norm: StructuredAudioNorm<B>,
@@ -91,7 +97,7 @@ pub(super) struct StructuredAudioConvNeXt<B: Backend> {
pub(super) pwconv2: StructuredAudioPointwiseConv<B>,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioResidualUnit<B: Backend> {
pub(super) snake1_alpha: Array<B>,
pub(super) conv1: StructuredAudioConv1d<B>,
@@ -99,7 +105,7 @@ pub(super) struct StructuredAudioResidualUnit<B: Backend> {
pub(super) conv2: StructuredAudioConv1d<B>,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioDecoderBlock<B: Backend> {
pub(super) snake_alpha: Array<B>,
pub(super) trans_conv: StructuredAudioConvTranspose1d<B>,
@@ -108,7 +114,7 @@ pub(super) struct StructuredAudioDecoderBlock<B: Backend> {
pub(super) res_unit3: StructuredAudioResidualUnit<B>,
}
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(super) struct StructuredAudioDecoderGraph<B: Backend> {
pub(super) first_conv: StructuredAudioConv1d<B>,
pub(super) upsample_blocks: Vec<(StructuredAudioConvTranspose1d<B>, StructuredAudioConvNeXt<B>)>,
diff --git a/crates/backend-uzu/src/backends/common/allocation_access.rs b/crates/backend-uzu/src/backends/common/allocation_access.rs
new file mode 100644
index 000000000..11e98e9cb
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/allocation_access.rs
@@ -0,0 +1,60 @@
+use thiserror::Error;
+
+use crate::{
+ ArrayElement,
+ backends::common::{Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, DenseBuffer},
+};
+
+#[derive(Debug, Clone, PartialEq, Eq, Error)]
+pub enum AllocationAccessError {
+ #[error("allocation write length {write_len} exceeds allocation length {range_len}")]
+ WriteExceedsRange {
+ write_len: usize,
+ range_len: usize,
+ },
+}
+
+pub fn allocation_copy_from_slice<B: Backend, T: ArrayElement>(
+ allocation: &mut Allocation<B>,
+ data: &[T],
+) -> Result<(), AllocationAccessError> {
+ let bytes = bytemuck::cast_slice(data);
+ if bytes.is_empty() {
+ return Ok(());
+ }
+ let destination = allocation_as_bytes_mut(allocation);
+ if bytes.len() > destination.len() {
+ return Err(AllocationAccessError::WriteExceedsRange {
+ write_len: bytes.len(),
+ range_len: destination.len(),
+ });
+ }
+ destination[..bytes.len()].copy_from_slice(bytes);
+ Ok(())
+}
+
+pub fn allocation_as_bytes<B: Backend>(allocation: &Allocation<B>) -> &[u8] {
+ let buffer_range = allocation.as_buffer_range_ref();
+ let range = buffer_range.range();
+ unsafe {
+ std::slice::from_raw_parts(
+ (buffer_range.buffer().cpu_ptr().as_ptr() as *const u8).add(range.start),
+ range.len(),
+ )
+ }
+}
+
+pub fn allocation_as_bytes_mut<B: Backend>(allocation: &mut Allocation<B>) -> &mut [u8] {
+ let buffer_range = allocation.as_buffer_range_mut();
+ let range = buffer_range.range();
+ unsafe {
+ std::slice::from_raw_parts_mut(
+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),
+ range.len(),
+ )
+ }
+}
+
+pub fn allocation_to_vec<B: Backend, T: ArrayElement>(allocation: &Allocation<B>) -> Vec<T> {
+ bytemuck::cast_slice(allocation_as_bytes(allocation)).to_vec()
+}
diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
index 1d50b5374..a24691f26 100644
--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs
+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs
@@ -90,6 +90,7 @@ impl<B: Backend> Allocator<B> {
size: usize,
allocation_type: AllocationType<B>,
) -> Result<Allocation<B>, B::Error> {
+ assert!(size > 0, "allocation size must be greater than 0");
let alignment = usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, 16_384);
let allocation_type = match allocation_type {
AllocationType::Global => RangeAllocationType::Global,
diff --git a/crates/backend-uzu/src/backends/common/buffer.rs b/crates/backend-uzu/src/backends/common/buffer.rs
index caa0f32d7..bd91dbb0e 100644
--- a/crates/backend-uzu/src/backends/common/buffer.rs
+++ b/crates/backend-uzu/src/backends/common/buffer.rs
@@ -57,6 +57,17 @@ impl<'a, B: Buffer> BufferRangeRef<'a, B> {
pub fn range(&self) -> Range<usize> {
self.range.clone()
}
+
+ pub(crate) fn subrange(
+ self,
+ range: Range<usize>,
+ ) -> Self {
+ assert!(range.end <= self.range.len(), "buffer subrange exceeds range");
+ Self {
+ buffer: self.buffer,
+ range: self.range.start + range.start..self.range.start + range.end,
+ }
+ }
}
pub trait AsBufferRangeRef {
@@ -98,6 +109,17 @@ impl<'a, B: Buffer> BufferRangeMut<'a, B> {
pub fn range(&self) -> Range<usize> {
self.range.clone()
}
+
+ pub(crate) fn subrange(
+ self,
+ range: Range<usize>,
+ ) -> Self {
+ assert!(range.end <= self.range.len(), "buffer subrange exceeds range");
+ Self {
+ buffer: self.buffer,
+ range: self.range.start + range.start..self.range.start + range.end,
+ }
+ }
}
pub trait AsBufferRangeMut: AsBufferRangeRef {
diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rs
index f636b18f4..7d6f9dd25 100644
--- a/crates/backend-uzu/src/backends/common/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs
@@ -1,6 +1,6 @@
-use std::{ops::Range, time::Duration};
+use std::time::Duration;
-use super::Backend;
+use super::{Backend, BufferRangeMut, BufferRangeRef};
pub trait CommandBuffer {
type Backend: Backend<CommandBuffer = Self>;
@@ -72,16 +72,13 @@ pub trait CommandBufferEncoding {
fn encode_copy(
&mut self,
- src: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer,
- src_range: Range<usize>,
- dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer,
- dst_range: Range<usize>,
+ src: BufferRangeRef<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,
+ dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,
);
fn encode_fill(
&mut self,
- dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer,
- range: Range<usize>,
+ dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,
value: u8,
);
diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rs
index f92080244..25018ebd9 100644
--- a/crates/backend-uzu/src/backends/common/encoder.rs
+++ b/crates/backend-uzu/src/backends/common/encoder.rs
@@ -1,12 +1,35 @@
-use std::{ops::Range, time::Duration};
+use std::{
+ ops::{Bound, Range, RangeBounds},
+ time::Duration,
+};
use crate::backends::common::{
- AccessFlags, Allocation, AllocationPool, AllocationType, Backend, BufferGpuAddressRangeExt, CommandBuffer,
- CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context,
+ AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend,
+ BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
+ CommandBufferInitial, CommandBufferPending, Context,
hazard_tracker::{Access, HazardTracker},
};
+fn resolve_copy_range(
+ range: impl RangeBounds<usize>,
+ allocation_len: usize,
+ label: &str,
+) -> Range<usize> {
+ let start = match range.start_bound() {
+ Bound::Included(&value) => value,
+ Bound::Excluded(&value) => value.checked_add(1).expect("copy range start overflow"),
+ Bound::Unbounded => 0,
+ };
+ let end = match range.end_bound() {
+ Bound::Included(&value) => value.checked_add(1).expect("copy range end overflow"),
+ Bound::Excluded(&value) => value,
+ Bound::Unbounded => allocation_len,
+ };
+ assert!(start <= end, "{label} copy range start exceeds end");
+ assert!(end <= allocation_len, "{label} copy range exceeds allocation");
+ start..end
+}
+
pub struct Encoder<'encoding, B: Backend> {
context: &'encoding B::Context,
command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,
@@ -58,37 +81,45 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
pub fn encode_copy(
&mut self,
- src: &B::DenseBuffer,
- src_range: Range<usize>,
- dst: &mut B::DenseBuffer,
- dst_range: Range<usize>,
+ src: &Allocation<B>,
+ src_range: impl RangeBounds<usize>,
+ dst: &mut Allocation<B>,
+ dst_range: impl RangeBounds<usize>,
) {
- let size = src_range.end - src_range.start;
- debug_assert_eq!(size, dst_range.end - dst_range.start);
+ let src_buffer_range = src.as_buffer_range_ref();
+ let dst_buffer_range = dst.as_buffer_range_mut();
+ let src_range = resolve_copy_range(src_range, src_buffer_range.range().len(), "source");
+ let dst_range = resolve_copy_range(dst_range, dst_buffer_range.range().len(), "destination");
+ let byte_len = src_range.len();
+ assert_eq!(byte_len, dst_range.len(), "copy range lengths must match");
+ assert!(byte_len > 0, "zero-sized copies are not allowed");
+ let src_buffer_range = src_buffer_range.subrange(src_range);
+ let dst_buffer_range = dst_buffer_range.subrange(dst_range);
self.access(&[
Access {
- range: src.gpu_address_subrange(src_range.clone()),
+ range: src_buffer_range.buffer().gpu_address_subrange(src_buffer_range.range()),
flags: AccessFlags::copy_read(),
},
Access {
- range: dst.gpu_address_subrange(dst_range.clone()),
+ range: dst_buffer_range.buffer().gpu_address_subrange(dst_buffer_range.range()),
flags: AccessFlags::copy_write(),
},
]);
- self.command_buffer.encode_copy(src, src_range, dst, dst_range);
+ self.command_buffer.encode_copy(src_buffer_range, dst_buffer_range);
}
pub fn encode_fill(
&mut self,
- dst: &mut B::DenseBuffer,
- range: Range<usize>,
+ dst: &mut Allocation<B>,
value: u8,
) {
+ let dst_buffer_range = dst.as_buffer_range_mut();
+ assert!(!dst_buffer_range.range().is_empty(), "zero-sized fills are not allowed");
self.access(&[Access {
- range: dst.gpu_address_subrange(range.clone()),
+ range: dst_buffer_range.buffer().gpu_address_subrange(dst_buffer_range.range()),
flags: AccessFlags::copy_write(),
}]);
- self.command_buffer.encode_fill(dst, range, value);
+ self.command_buffer.encode_fill(dst_buffer_range, value);
}
pub fn access(
@@ -127,6 +158,10 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
&mut self.command_buffer
}
+ pub fn context(&self) -> &'encoding B::Context {
+ self.context
+ }
+
pub fn end_encoding(self) -> Executable<B> {
Executable {
command_buffer: self.command_buffer.end_encoding(),
diff --git a/crates/backend-uzu/src/backends/common/kernel/attention.rs b/crates/backend-uzu/src/backends/common/kernel/attention.rs
index c56663580..5fcf577c1 100644
--- a/crates/backend-uzu/src/backends/common/kernel/attention.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/attention.rs
@@ -6,7 +6,7 @@ use std::{
use crate::{
DataType,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
gpu_types::{AttnParams, ring::RingParams},
kernel::AttentionGemmKernel,
},
@@ -15,12 +15,12 @@ use crate::{
const BQ: usize = 32;
pub struct AttentionGemmArguments<'a, B: Backend> {
- pub queries_buffer: &'a B::DenseBuffer,
- pub keys_buffer: &'a B::DenseBuffer,
- pub values_buffer: &'a B::DenseBuffer,
- pub output_buffer: &'a mut B::DenseBuffer,
- pub trie_buffer: Option<&'a B::DenseBuffer>,
- pub sinks_buffer: Option<&'a B::DenseBuffer>,
+ pub queries: &'a Allocation<B>,
+ pub keys: &'a Allocation<B>,
+ pub values: &'a Allocation<B>,
+ pub output: &'a mut Allocation<B>,
+ pub trie: Option<&'a Allocation<B>>,
+ pub sinks: Option<&'a Allocation<B>>,
pub num_heads: usize,
pub num_groups: usize,
pub suffix_length: usize, // qL
@@ -58,7 +58,6 @@ impl<B: Backend> AttentionGemmBlock<B> {
pub fn encode(
&self,
- context: &B::Context,
encoder: &mut Encoder<B>,
args: AttentionGemmArguments<B>,
) -> Result<(), B::Error> {
@@ -72,9 +71,9 @@ impl<B: Backend> AttentionGemmBlock<B> {
let align_k = (args.sequence_length % bk) == 0;
let is_kv_cache_ring = args.ring_params.is_some();
let is_causal = args.is_causal;
- let is_trie = args.trie_buffer.is_some();
+ let is_trie = args.trie.is_some();
let is_sliding_window = args.sliding_window_size.is_some();
- let has_sinks = args.sinks_buffer.is_some();
+ let has_sinks = args.sinks.is_some();
let key = KernelKey {
bk,
head_dim,
@@ -92,7 +91,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
let kernel = <B::Kernels as Kernels>::AttentionGemmKernel::new(
- context,
+ encoder.context(),
self.data_type,
bk as u32,
head_dim as u32,
@@ -137,15 +136,15 @@ impl<B: Backend> AttentionGemmBlock<B> {
};
kernel.encode(
- args.queries_buffer,
- args.keys_buffer,
- args.values_buffer,
- args.output_buffer,
+ args.queries,
+ args.keys,
+ args.values,
+ args.output,
params,
args.ring_params,
- args.trie_buffer,
+ args.trie,
args.sliding_window_size.map(|s| s as u32),
- args.sinks_buffer,
+ args.sinks,
args.num_heads as u32,
args.suffix_length as u32,
encoder,
diff --git a/crates/backend-uzu/src/backends/common/kernel/audio.rs b/crates/backend-uzu/src/backends/common/kernel/audio.rs
index f7be333b6..dc4e3aa63 100644
--- a/crates/backend-uzu/src/backends/common/kernel/audio.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/audio.rs
@@ -7,9 +7,9 @@ pub enum AudioPadMode {
}
pub struct AudioFsqDecodeArguments<'a, B: Backend> {
- pub tokens: &'a B::Buffer,
- pub lengths: &'a B::Buffer,
- pub output: &'a B::Buffer,
+ pub tokens: &'a B::DenseBuffer,
+ pub lengths: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
pub batch_size: usize,
pub num_groups: usize,
pub seq_len: usize,
@@ -18,9 +18,9 @@ pub struct AudioFsqDecodeArguments<'a, B: Backend> {
}
pub struct AudioFsqEncodeArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub tokens: &'a B::Buffer,
- pub lengths: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub tokens: &'a B::DenseBuffer,
+ pub lengths: &'a B::DenseBuffer,
pub batch_size: usize,
pub num_groups: usize,
pub seq_len: usize,
@@ -31,11 +31,11 @@ pub struct AudioFsqEncodeArguments<'a, B: Backend> {
}
pub struct AudioConv1dArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub weight: &'a B::Buffer,
- pub bias: &'a B::Buffer,
- pub output: &'a B::Buffer,
- pub lengths: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub weight: &'a B::DenseBuffer,
+ pub bias: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
+ pub lengths: &'a B::DenseBuffer,
pub batch_size: usize,
pub cin: usize,
pub cout: usize,
@@ -49,11 +49,11 @@ pub struct AudioConv1dArguments<'a, B: Backend> {
}
pub struct AudioCausalConv1dArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub weight: &'a B::Buffer,
- pub bias: &'a B::Buffer,
- pub output: &'a B::Buffer,
- pub lengths: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub weight: &'a B::DenseBuffer,
+ pub bias: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
+ pub lengths: &'a B::DenseBuffer,
pub batch_size: usize,
pub cin: usize,
pub cout: usize,
@@ -63,11 +63,11 @@ pub struct AudioCausalConv1dArguments<'a, B: Backend> {
}
pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub weight: &'a B::Buffer,
- pub bias: &'a B::Buffer,
- pub output: &'a B::Buffer,
- pub lengths: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub weight: &'a B::DenseBuffer,
+ pub bias: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
+ pub lengths: &'a B::DenseBuffer,
pub batch_size: usize,
pub cin: usize,
pub cout: usize,
@@ -78,9 +78,9 @@ pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {
}
pub struct AudioHalfSnakeArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub alpha: &'a B::Buffer,
- pub output: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub alpha: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
pub batch_size: usize,
pub channels: usize,
pub seq_len: usize,
@@ -90,28 +90,28 @@ pub struct AudioHalfSnakeArguments<'a, B: Backend> {
}
pub struct AudioElementwiseArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub output: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
pub n: usize,
}
pub struct AudioAddArguments<'a, B: Backend> {
- pub a: &'a B::Buffer,
- pub b: &'a B::Buffer,
- pub output: &'a B::Buffer,
+ pub a: &'a B::DenseBuffer,
+ pub b: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
pub n: usize,
}
pub struct AudioScaleArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub output: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
pub n: usize,
pub scale: f32,
}
pub struct AudioClampArguments<'a, B: Backend> {
- pub input: &'a B::Buffer,
- pub output: &'a B::Buffer,
+ pub input: &'a B::DenseBuffer,
+ pub output: &'a B::DenseBuffer,
pub n: usize,
pub min_value: f32,
pub max_value: f32,
@@ -121,67 +121,67 @@ pub trait AudioKernelRuntime<B: Backend> {
fn encode_fsq_decode(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioFsqDecodeArguments<'_, B>,
+ arguments: AudioFsqDecodeArguments<B>,
) -> Result<(), B::Error>;
fn encode_fsq_encode(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioFsqEncodeArguments<'_, B>,
+ arguments: AudioFsqEncodeArguments<B>,
) -> Result<(), B::Error>;
fn encode_conv1d(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioConv1dArguments<'_, B>,
+ arguments: AudioConv1dArguments<B>,
) -> Result<(), B::Error>;
fn encode_causal_conv1d(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioCausalConv1dArguments<'_, B>,
+ arguments: AudioCausalConv1dArguments<B>,
) -> Result<(), B::Error>;
fn encode_causal_conv_transpose1d(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioCausalConvTranspose1dArguments<'_, B>,
+ arguments: AudioCausalConvTranspose1dArguments<B>,
) -> Result<(), B::Error>;
fn encode_half_snake(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioHalfSnakeArguments<'_, B>,
+ arguments: AudioHalfSnakeArguments<B>,
) -> Result<(), B::Error>;
fn encode_leaky_relu(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioElementwiseArguments<'_, B>,
+ arguments: AudioElementwiseArguments<B>,
negative_slope: f32,
) -> Result<(), B::Error>;
fn encode_tanh(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioElementwiseArguments<'_, B>,
+ arguments: AudioElementwiseArguments<B>,
) -> Result<(), B::Error>;
fn encode_add(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioAddArguments<'_, B>,
+ arguments: AudioAddArguments<B>,
) -> Result<(), B::Error>;
fn encode_scale(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioScaleArguments<'_, B>,
+ arguments: AudioScaleArguments<B>,
) -> Result<(), B::Error>;
fn encode_clamp(
&self,
encoder: &mut Encoder<B>,
- arguments: AudioClampArguments<'_, B>,
+ arguments: AudioClampArguments<B>,
) -> Result<(), B::Error>;
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs
index 0c28fc409..6adb42def 100644
--- a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -1,16 +1,16 @@
-use std::{cell::RefCell, mem::size_of, ops::DerefMut, rc::Rc};
+use std::mem::size_of;
use thiserror::Error;
use crate::{
DataType,
- backends::common::{Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
+ backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
};
-pub struct KVLayerData<B: Backend> {
- pub key_buffer: Rc<RefCell<B::DenseBuffer>>,
+pub struct KVLayerData<'a, B: Backend> {
+ pub key_allocation: &'a mut Allocation<B>,
pub key_shape: [usize; 3],
- pub value_buffer: Rc<RefCell<B::DenseBuffer>>,
+ pub value_allocation: &'a mut Allocation<B>,
pub value_shape: [usize; 3],
}
@@ -55,7 +55,7 @@ impl<B: Backend> KVCacheUpdate<B> {
/// Encode the KV cache update operation using a provided command buffer
pub fn encode(
&self,
- in_place_data: &[KVLayerData<B>],
+ in_place_data: &mut [KVLayerData<B>],
source_indices: &[usize],
destination_indices: &[usize],
encoder: &mut Encoder<B>,
@@ -80,8 +80,8 @@ impl<B: Backend> KVCacheUpdate<B> {
// non-inline is not supported yet (and is broken anyways due to a data race)
for swaps_chunk in swaps.chunks(max_inline_swaps) {
self.kernel.encode(
- layer_data.key_buffer.borrow_mut().deref_mut(),
- layer_data.value_buffer.borrow_mut().deref_mut(),
+ &mut *layer_data.key_allocation,
+ &mut *layer_data.value_allocation,
swaps_chunk,
swaps_chunk.len() as u32,
num_heads as u32,
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rs
index 6e1008853..e8ef8c962 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul.rs
@@ -2,7 +2,7 @@ use thiserror::Error;
use crate::{
DataType,
- backends::common::{Backend, Encoder, kernel::ManualKernels},
+ backends::common::{Allocation, Backend, Encoder, kernel::ManualKernels},
};
#[derive(Debug, Error)]
@@ -13,29 +13,27 @@ pub enum MatmulError<B: Backend> {
BackendError(#[source] B::Error),
}
-#[derive(Debug)]
pub enum MatmulArgumentC<'a, B: Backend> {
None,
/// Accumulate: [M, N]
Accumulate,
/// Bias: [N] (broadcasted across M/batch)
- Bias(&'a B::DenseBuffer),
+ Bias(&'a Allocation<B>),
}
// D = ab_scale * (A @ B.T) + C
-#[derive(Debug)]
pub struct MatmulArguments<'a, B: Backend> {
/// A: [M, K]
- pub a: &'a B::DenseBuffer,
- pub a_offset: u64,
+ pub a: &'a Allocation<B>,
+ pub a_offset: usize,
/// B: [N, K]
- pub b: &'a B::DenseBuffer,
+ pub b: &'a Allocation<B>,
/// AB scale: also known as alpha
pub ab_scale: f32,
/// C: behavior depends on enum variant
pub c: MatmulArgumentC<'a, B>,
/// D: [M, N]
- pub d: &'a mut B::DenseBuffer,
+ pub d: &'a mut Allocation<B>,
/// M dimension: usually batch/number of tokens (rows of A, rows of C)
pub batch_dim: u32,
/// K dimension: usually input_dim/reduction dimension (cols of A, rows of B)
@@ -54,7 +52,6 @@ pub trait MatmulKernel: Sized {
fn encode(
&mut self,
- context: &<Self::Backend as Backend>::Context,
arguments: MatmulArguments<Self::Backend>,
encoder: &mut Encoder<Self::Backend>,
);
diff --git a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
index 1c4784f9e..8676b3e1e 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -1,7 +1,7 @@
use crate::{
DataType,
backends::common::{
- ActivationConfig, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
+ ActivationConfig, Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
},
};
@@ -9,7 +9,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {
kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,
activation: ActivationConfig,
hidden_dim: usize,
- hadamard_factors: Option<B::DenseBuffer>,
+ hadamard_factors: Option<Allocation<B>>,
}
impl<B: Backend> MlpGateActMulEncodable<B> {
@@ -18,7 +18,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
data_type: DataType,
activation: ActivationConfig,
hidden_dim: usize,
- hadamard_factors: Option<B::DenseBuffer>,
+ hadamard_factors: Option<Allocation<B>>,
) -> Result<Self, B::Error> {
let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type, hadamard_factors.is_some())?;
Ok(Self {
@@ -32,8 +32,8 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
pub fn encode(
&self,
encoder: &mut Encoder<B>,
- fused_up: &B::DenseBuffer,
- hidden: &mut B::DenseBuffer,
+ fused_up: &Allocation<B>,
+ hidden: &mut Allocation<B>,
m: i32,
) -> Result<(), B::Error> {
if self.activation.act_type() == ActivationType::IDENTITY {
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rs
index fe30a1349..30ce0effd 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rs
@@ -1,9 +1,8 @@
-use std::ops::{Deref, DerefMut};
-
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{MoeExpertsDecodeSinglePassAKernel, MoeExpertsDecodeSinglePassBKernel},
},
};
@@ -11,26 +10,21 @@ use crate::{
static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
/// Arguments for single-token MoE decode (T=1 optimized path)
-#[derive(Debug)]
pub struct MoeExpertsSingleDecodeArguments<'a, B: Backend> {
/// Input activation [d_model]
- pub x: &'a B::DenseBuffer,
+ pub x: &'a Allocation<B>,
/// Top-K expert indices from router [K]
- pub topk_ids: &'a B::DenseBuffer,
+ pub topk_ids: &'a Allocation<B>,
/// Top-K probabilities from router [K]
- pub topk_probs: &'a B::DenseBuffer,
+ pub topk_probs: &'a Allocation<B>,
/// Up/gate projection weights [E, 2*d_ff, d_model]
- pub w13_all: &'a B::DenseBuffer,
+ pub w13_all: &'a Allocation<B>,
/// Down projection weights [E, d_model, d_ff]
- pub w2_all: &'a B::DenseBuffer,
+ pub w2_all: &'a Allocation<B>,
/// Up/gate biases [E, 2*d_ff]
- pub up_biases: &'a B::DenseBuffer,
+ pub up_biases: &'a Allocation<B>,
/// Down biases [E, d_model]
- pub down_biases: &'a B::DenseBuffer,
- /// Hidden buffer [K, d_ff] - intermediate storage (f32)
- pub hidden: &'a mut B::DenseBuffer,
- /// Final output [d_model]
- pub y: &'a mut B::DenseBuffer,
+ pub down_biases: &'a Allocation<B>,
/// Model dimension
pub d_model: usize,
/// FFN hidden dimension
@@ -85,14 +79,14 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
pub fn encode(
&self,
encoder: &mut Encoder<B>,
- mut args: MoeExpertsSingleDecodeArguments<B>,
- ) {
- if args.k == 0 {
- return;
- }
+ args: MoeExpertsSingleDecodeArguments<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ assert!(args.k > 0, "MoE single decode requires at least one active expert");
let gate_idx = args.gating_code.min(3) as usize;
let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.k, args.d_ff], DataType::F32))?;
+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.d_model], args.data_type))?;
// Pass A: x @ W13[expert] -> hidden
let kernel = &self.pass_a[gate_idx][dtype_idx];
@@ -101,7 +95,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
args.topk_ids,
args.w13_all,
args.up_biases,
- args.hidden.deref_mut(),
+ &mut hidden,
args.d_model as u32,
args.d_ff as u32,
args.k as u32,
@@ -116,16 +110,17 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
// Pass B: 8 simdgroups (256 threads), outputs final y directly
let kernel = &self.pass_b[dtype_idx];
kernel.encode(
- args.hidden.deref(),
+ &hidden,
args.topk_ids,
args.topk_probs,
args.w2_all,
args.down_biases,
- args.y,
+ &mut output,
args.d_model as u32,
args.d_ff as u32,
args.k as u32,
encoder,
);
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
index 0749d385e..8fc689c81 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
@@ -1,13 +1,12 @@
-use std::ops::{Deref, DerefMut};
-
use super::{
MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,
MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,
};
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},
},
};
@@ -48,11 +47,10 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
pub fn encode(
&self,
encoder: &mut Encoder<B>,
- mut args: MoeExpertsTwoPassArguments<B>,
- ) {
- if args.total_rows == 0 {
- return;
- }
+ args: MoeExpertsTwoPassArguments<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;
// pass a tile
const BLOCK_M: u32 = 4;
@@ -61,7 +59,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
encoder,
MoePassATileCountsArguments {
expert_offsets: args.expert_offsets,
- tile_counts: args.tile_counts.deref_mut(),
+ tile_counts: &mut *args.tile_counts,
e: args.e,
h_blocks,
},
@@ -69,9 +67,9 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
self.pass_a_tile.encode_scan(
encoder,
MoePassATileScanArguments {
- tile_counts: args.tile_counts,
- tile_offsets: args.tile_offsets.deref_mut(),
- total_tiles: args.total_tiles.deref_mut(),
+ tile_counts: &*args.tile_counts,
+ tile_offsets: &mut *args.tile_offsets,
+ total_tiles: &mut *args.total_tiles,
e: args.e,
},
);
@@ -79,7 +77,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
encoder,
MoePassARowMapArguments {
expert_offsets: args.expert_offsets,
- row_expert_map: args.row_expert_map.deref_mut(),
+ row_expert_map: &mut *args.row_expert_map,
total_rows: args.total_rows,
e: args.e,
},
@@ -88,9 +86,9 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
encoder,
MoePassATileBuildArguments {
expert_offsets: args.expert_offsets,
- tile_offsets: args.tile_offsets,
- row_expert_map: args.row_expert_map,
- tile_map: args.tile_map.deref_mut(),
+ tile_offsets: &*args.tile_offsets,
+ row_expert_map: &*args.row_expert_map,
+ tile_map: &mut *args.tile_map,
total_rows: args.total_rows,
h_blocks,
},
@@ -98,8 +96,8 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
self.pass_a_tile.encode_dispatch_args(
encoder,
MoePassATileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args.deref_mut(),
+ total_tiles: &*args.total_tiles,
+ dispatch_args: &mut *args.dispatch_args,
num_tiles_y: 1,
},
);
@@ -110,10 +108,10 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
// pass a
let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
pass_a_kernel.encode(
- args.x_perm_buffer,
+ args.x_perm,
args.expert_offsets,
args.w13_all,
- args.hidden_buffer.deref_mut(),
+ &mut hidden,
args.up_biases,
args.d_model as u32,
args.d_ff as u32,
@@ -123,24 +121,25 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
args.up_clip_min,
args.up_clip_max,
args.silu_alpha,
- args.tile_map.deref(),
- args.dispatch_args.deref(),
+ &*args.tile_map,
+ &*args.dispatch_args,
encoder,
);
// pass b
let pass_b_kernel = &self.fused_down[dtype_idx];
pass_b_kernel.encode(
- args.hidden_buffer.deref(),
- args.row_expert_map.deref(),
+ &hidden,
+ &*args.row_expert_map,
args.w2_all,
args.down_biases,
- args.output_buffer.deref_mut(),
+ &mut output,
args.total_rows as u32,
args.d_model as u32,
args.d_ff as u32,
args.e as u32,
encoder,
);
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
index 25a781aa2..c4f65ef6e 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
@@ -1,12 +1,11 @@
-use std::ops::{Deref, DerefMut};
-
use super::{
MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,
};
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},
},
};
@@ -49,29 +48,26 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
pub fn encode(
&self,
encoder: &mut Encoder<B>,
- mut args: MoeExpertsTwoPassArguments<B>,
- ) {
- if args.total_rows == 0 {
- return;
- }
-
- let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
- encoder.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);
+ args: MoeExpertsTwoPassArguments<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;
+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;
+ encoder.encode_fill(&mut hidden, 0);
self.tile_map.encode_counts(
encoder,
MoeTileCountsArguments {
- offsets_buffer: args.expert_offsets,
- tile_counts_buffer: args.tile_counts.deref_mut(),
+ offsets: args.expert_offsets,
+ tile_counts: &mut *args.tile_counts,
e: args.e,
},
);
self.tile_map.encode_scan(
encoder,
MoeTileScanArguments {
- tile_counts_buffer: args.tile_counts,
- tile_offsets_buffer: args.tile_offsets.deref_mut(),
- total_tiles_buffer: args.total_tiles.deref_mut(),
+ tile_counts: &*args.tile_counts,
+ tile_offsets: &mut *args.tile_offsets,
+ total_tiles: &mut *args.total_tiles,
e: args.e,
},
);
@@ -79,9 +75,9 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
encoder,
MoeTileMapBuildArguments {
expert_offsets: args.expert_offsets,
- tile_offsets: args.tile_offsets,
- tile_counts: args.tile_counts,
- tile_map: args.tile_map.deref_mut(),
+ tile_offsets: &*args.tile_offsets,
+ tile_counts: &*args.tile_counts,
+ tile_map: &mut *args.tile_map,
e: args.e,
},
);
@@ -102,8 +98,8 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
self.tile_map.encode_dispatch_args(
encoder,
MoeTileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args.deref_mut(),
+ total_tiles: &*args.total_tiles,
+ dispatch_args: &mut *args.dispatch_args,
num_tiles_x: n_tiles_ff,
},
);
@@ -113,11 +109,11 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];
kernel_pass_a.encode(
- args.x_perm_buffer,
+ args.x_perm,
args.expert_offsets,
args.w13_all,
args.up_biases,
- args.hidden_buffer.deref_mut(),
+ &mut hidden,
args.d_model as u32,
args.d_ff as u32,
args.e as u32,
@@ -126,51 +122,49 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args.up_clip_min,
args.up_clip_max,
args.silu_alpha,
- args.tile_map.deref(),
- args.dispatch_args.deref(),
+ &*args.tile_map,
+ &*args.dispatch_args,
encoder,
);
let dispatch_args = MoeTileDispatchArguments {
- total_tiles: args.total_tiles,
- dispatch_args: args.dispatch_args.deref_mut(),
+ total_tiles: &*args.total_tiles,
+ dispatch_args: &mut *args.dispatch_args,
num_tiles_x: n_tiles_model,
};
self.tile_map.encode_dispatch_args(encoder, dispatch_args);
let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
kernel_pass_b.encode(
- args.hidden_buffer.deref(),
+ &hidden,
args.expert_offsets,
args.w2_all,
args.down_biases,
- args.output_buffer.deref_mut(),
+ &mut output,
args.d_model as u32,
args.d_ff as u32,
args.e as u32,
- args.tile_map.deref(),
- args.dispatch_args.deref(),
+ &*args.tile_map,
+ &*args.dispatch_args,
encoder,
);
+ Ok(output)
}
}
-#[derive(Debug)]
pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {
- pub x_perm_buffer: &'a B::DenseBuffer,
- pub expert_offsets: &'a B::DenseBuffer,
- pub row_expert_map: &'a mut B::DenseBuffer,
- pub hidden_buffer: &'a mut B::DenseBuffer,
- pub output_buffer: &'a mut B::DenseBuffer,
- pub w13_all: &'a B::DenseBuffer,
- pub w2_all: &'a B::DenseBuffer,
- pub up_biases: &'a B::DenseBuffer,
- pub down_biases: &'a B::DenseBuffer,
- pub tile_counts: &'a mut B::DenseBuffer,
- pub tile_offsets: &'a mut B::DenseBuffer,
- pub tile_map: &'a mut B::DenseBuffer,
- pub total_tiles: &'a mut B::DenseBuffer,
- pub dispatch_args: &'a mut B::DenseBuffer,
+ pub x_perm: &'a Allocation<B>,
+ pub expert_offsets: &'a Allocation<B>,
+ pub row_expert_map: &'a mut Allocation<B>,
+ pub w13_all: &'a Allocation<B>,
+ pub w2_all: &'a Allocation<B>,
+ pub up_biases: &'a Allocation<B>,
+ pub down_biases: &'a Allocation<B>,
+ pub tile_counts: &'a mut Allocation<B>,
+ pub tile_offsets: &'a mut Allocation<B>,
+ pub tile_map: &'a mut Allocation<B>,
+ pub total_tiles: &'a mut Allocation<B>,
+ pub dispatch_args: &'a mut Allocation<B>,
pub total_rows: usize,
pub d_model: usize,
pub d_ff: usize,
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs
index fc6887896..9c6040b37 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs
@@ -1,17 +1,16 @@
use crate::{
DataType,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
},
};
-#[derive(Debug)]
pub struct MoeGatherArguments<'a, B: Backend> {
- pub x_buffer: &'a B::DenseBuffer,
- pub bucketed_ids_buffer: &'a B::DenseBuffer,
- pub x_perm_buffer: &'a mut B::DenseBuffer,
- pub sumk_buffer: &'a B::DenseBuffer,
+ pub x: &'a Allocation<B>,
+ pub bucketed_ids: &'a Allocation<B>,
+ pub x_perm: &'a mut Allocation<B>,
+ pub sumk: &'a Allocation<B>,
pub t: usize,
pub k: usize,
pub d_model: usize,
@@ -40,30 +39,30 @@ impl<B: Backend> MoeGatherKernels<B> {
) {
match dtype {
DataType::F32 => self.f32.encode(
- args.x_buffer,
- args.bucketed_ids_buffer,
- args.x_perm_buffer,
- args.sumk_buffer,
+ args.x,
+ args.bucketed_ids,
+ args.x_perm,
+ args.sumk,
args.d_model as u32,
args.t as u32,
args.k as u32,
encoder,
),
DataType::F16 => self.f16.encode(
- args.x_buffer,
- args.bucketed_ids_buffer,
- args.x_perm_buffer,
- args.sumk_buffer,
+ args.x,
+ args.bucketed_ids,
+ args.x_perm,
+ args.sumk,
args.d_model as u32,
args.t as u32,
args.k as u32,
encoder,
),
DataType::BF16 => self.bf16.encode(
- args.x_buffer,
- args.bucketed_ids_buffer,
- args.x_perm_buffer,
- args.sumk_buffer,
+ args.x,
+ args.bucketed_ids,
+ args.x_perm,
+ args.sumk,
args.d_model as u32,
args.t as u32,
args.k as u32,
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs
index 3bb9871fc..6bff799cc 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs
@@ -1,5 +1,5 @@
use crate::backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},
};
@@ -25,7 +25,7 @@ impl<B: Backend> MoeTileMapKernels<B> {
encoder: &mut Encoder<B>,
args: MoeTileCountsArguments<B>,
) {
- self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);
+ self.counts.encode(args.offsets, args.tile_counts, args.e as u32, encoder);
}
pub fn encode_scan(
@@ -33,13 +33,7 @@ impl<B: Backend> MoeTileMapKernels<B> {
encoder: &mut Encoder<B>,
args: MoeTileScanArguments<B>,
) {
- self.scan.encode(
- args.tile_counts_buffer,
- args.tile_offsets_buffer,
- args.total_tiles_buffer,
- args.e as u32,
- encoder,
- );
+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);
}
pub fn encode_build_map(
@@ -67,30 +61,28 @@ impl<B: Backend> MoeTileMapKernels<B> {
}
pub struct MoeTileCountsArguments<'a, B: Backend> {
- pub offsets_buffer: &'a B::DenseBuffer, // [E+1]
- pub tile_counts_buffer: &'a mut B::DenseBuffer, // [E]
+ pub offsets: &'a Allocation<B>, // [E+1]
+ pub tile_counts: &'a mut Allocation<B>, // [E]
pub e: usize,
}
pub struct MoeTileScanArguments<'a, B: Backend> {
- pub tile_counts_buffer: &'a B::DenseBuffer, // [E]
- pub tile_offsets_buffer: &'a mut B::DenseBuffer, // [E+1]
- pub total_tiles_buffer: &'a mut B::DenseBuffer, // [>=2]
+ pub tile_counts: &'a Allocation<B>, // [E]
+ pub tile_offsets: &'a mut Allocation<B>, // [E+1]
+ pub total_tiles: &'a mut Allocation<B>, // [>=2]
pub e: usize,
}
-#[derive(Debug)]
pub struct MoeTileMapBuildArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::DenseBuffer, // [E+1]
- pub tile_offsets: &'a B::DenseBuffer, // [E+1]
- pub tile_counts: &'a B::DenseBuffer, // [E]
- pub tile_map: &'a mut B::DenseBuffer, // [total_tiles * 3]
+ pub expert_offsets: &'a Allocation<B>, // [E+1]
+ pub tile_offsets: &'a Allocation<B>, // [E+1]
+ pub tile_counts: &'a Allocation<B>, // [E]
+ pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]
pub e: usize,
}
-#[derive(Debug)]
pub struct MoeTileDispatchArguments<'a, B: Backend> {
- pub total_tiles: &'a B::DenseBuffer, // [>=1]
- pub dispatch_args: &'a mut B::DenseBuffer, // [3]
- pub num_tiles_x: u32, // x dimension for indirect dispatch
+ pub total_tiles: &'a Allocation<B>, // [>=1]
+ pub dispatch_args: &'a mut Allocation<B>, // [3]
+ pub num_tiles_x: u32, // x dimension for indirect dispatch
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
index 69ee10cf3..cc3f8674e 100644
--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
@@ -1,49 +1,44 @@
use crate::backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{
Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,
MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,
},
};
-#[derive(Debug)]
pub struct MoePassATileCountsArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::DenseBuffer, // [E+1]
- pub tile_counts: &'a mut B::DenseBuffer, // [E]
+ pub expert_offsets: &'a Allocation<B>, // [E+1]
+ pub tile_counts: &'a mut Allocation<B>, // [E]
pub e: usize,
pub h_blocks: u32,
}
-#[derive(Debug)]
pub struct MoePassATileScanArguments<'a, B: Backend> {
- pub tile_counts: &'a B::DenseBuffer, // [E]
- pub tile_offsets: &'a mut B::DenseBuffer, // [E+1]
- pub total_tiles: &'a mut B::DenseBuffer, // [>=1]
+ pub tile_counts: &'a Allocation<B>, // [E]
+ pub tile_offsets: &'a mut Allocation<B>, // [E+1]
+ pub total_tiles: &'a mut Allocation<B>, // [>=1]
pub e: usize,
}
-#[derive(Debug)]
pub struct MoePassARowMapArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::DenseBuffer, // [E+1]
- pub row_expert_map: &'a mut B::DenseBuffer, // [total_rows]
+ pub expert_offsets: &'a Allocation<B>, // [E+1]
+ pub row_expert_map: &'a mut Allocation<B>, // [total_rows]
pub total_rows: usize,
pub e: usize,
}
-#[derive(Debug)]
pub struct MoePassATileBuildArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::DenseBuffer, // [E+1]
- pub tile_offsets: &'a B::DenseBuffer, // [E+1]
- pub row_expert_map: &'a B::DenseBuffer, // [total_rows]
- pub tile_map: &'a mut B::DenseBuffer, // [total_tiles * 3]
+ pub expert_offsets: &'a Allocation<B>, // [E+1]
+ pub tile_offsets: &'a Allocation<B>, // [E+1]
+ pub row_expert_map: &'a Allocation<B>, // [total_rows]
+ pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]
pub total_rows: usize,
pub h_blocks: u32,
}
-#[derive(Debug)]
pub struct MoePassATileDispatchArguments<'a, B: Backend> {
- pub total_tiles: &'a B::DenseBuffer, // [>=1]
- pub dispatch_args: &'a mut B::DenseBuffer, // [3]
+ pub total_tiles: &'a Allocation<B>, // [>=1]
+ pub dispatch_args: &'a mut Allocation<B>, // [3]
pub num_tiles_y: u32,
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
index 4320f290b..af80634fe 100644
--- a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs
@@ -1,7 +1,7 @@
use crate::{
DataType,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
gpu_types::QuantizationMode,
kernel::{QuantizedMatmulQmmTransposedKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},
},
@@ -37,13 +37,13 @@ pub struct QuantizedMatmulConfiguration {
}
pub struct QuantizedMatmulArguments<'a, B: Backend> {
- pub a_buffer: &'a B::DenseBuffer,
+ pub a: &'a Allocation<B>,
pub a_offset: usize,
- pub b_buffer: &'a B::DenseBuffer,
- pub scales_buffer: &'a B::DenseBuffer,
- pub zero_points_or_biases_buffer: &'a B::DenseBuffer,
- pub output_buffer: &'a mut B::DenseBuffer,
- pub hadamard_factors: Option<&'a B::DenseBuffer>,
+ pub b: &'a Allocation<B>,
+ pub scales: &'a Allocation<B>,
+ pub zero_points_or_biases: &'a Allocation<B>,
+ pub output: &'a mut Allocation<B>,
+ pub hadamard_factors: Option<&'a Allocation<B>>,
pub batch_dim: usize,
}
@@ -62,7 +62,6 @@ enum MatrixVectorKernel<B: Backend> {
type QmmKernel<B> = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel;
-/// Big tile alone, or paired with a BM=8 small tile for `batch_dim < 48`.
enum MatrixMatrixKernel<B: Backend> {
Big(QmmKernel<B>),
BigAndSmall {
@@ -72,7 +71,6 @@ enum MatrixMatrixKernel<B: Backend> {
}
impl<B: Backend> MatrixMatrixKernel<B> {
- /// `None` means fall back to `matrix_vector` (small `batch_dim`, no small tile available).
fn pick(
&self,
batch_dim: usize,
@@ -112,7 +110,6 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);
let use_zero_points = !use_mlx_quant;
- // Matrix-vector
let matrix_vector = if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {
MatrixVectorKernel::QmvFast(
<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
@@ -144,8 +141,6 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
)
};
- // 64-tile (BM=BN=64): bf16 + output_dim%64==0; BK=64 if group_size≥64, BK=32 otherwise.
- // 32-tile fallback otherwise.
let aligned_n_64 = configuration.output_dim % 64 == 0;
let aligned_n_32 = configuration.output_dim % 32 == 0;
let is_bf16 = configuration.data_type == DataType::BF16;
@@ -167,8 +162,8 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
bm,
bk,
bn,
- 2u32, // WM
- 2u32, // WN
+ 2u32,
+ 2u32,
use_zero_points,
use_mlx_quant,
configuration.use_hadamard,
@@ -176,23 +171,21 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
)
.map_err(QuantizedMatmulError::BackendError)?;
- // Small-tile kernel (BM=8): batch_dim in [8, 48). Skipped for hadamard configs
- // since the BM=8 tile doesn't support it; QmvFast handles those batch sizes instead.
let matrix_matrix = if aligned_n_32 && !configuration.use_hadamard {
let small = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
context,
configuration.data_type,
group_size,
bits,
- 8u32, // BM
- 32u32, // BK
- 32u32, // BN
- 1u32, // WM
- 1u32, // WN
+ 8u32,
+ 32u32,
+ 32u32,
+ 1u32,
+ 1u32,
use_zero_points,
use_mlx_quant,
- false, // use_hadamard: BM=8 tile does not support hadamard
- true, // aligned_n: BN=32, always aligned for the small tile
+ false,
+ true,
)
.map_err(QuantizedMatmulError::BackendError)?;
MatrixMatrixKernel::BigAndSmall {
@@ -216,45 +209,52 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
&self,
encoder: &mut Encoder<B>,
arguments: QuantizedMatmulArguments<B>,
- ) -> Result<(), QuantizedMatmulError<B>> {
+ ) {
+ let QuantizedMatmulArguments {
+ a,
+ a_offset,
+ b,
+ scales,
+ zero_points_or_biases,
+ output,
+ hadamard_factors,
+ batch_dim,
+ } = arguments;
+
let (zero_points, biases) = match self.quantization_type {
- QuantizedMatmulType::ZeroPoint => (Some(arguments.zero_points_or_biases_buffer), None),
- QuantizedMatmulType::Mlx => (None, Some(arguments.zero_points_or_biases_buffer)),
+ QuantizedMatmulType::ZeroPoint => (Some(zero_points_or_biases), None),
+ QuantizedMatmulType::Mlx => (None, Some(zero_points_or_biases)),
};
macro_rules! encode_kernel {
- ($kernel:expr, $($hadamard:expr)?) => {
+ ($kernel:expr $(, $hadamard:expr)?) => {
$kernel.encode(
- arguments.b_buffer,
- arguments.scales_buffer,
+ b,
+ scales,
zero_points,
biases,
- (arguments.a_buffer, arguments.a_offset),
- arguments.output_buffer,
+ (a, a_offset),
+ output,
$($hadamard,)?
self.input_dim as u32,
self.output_dim as u32,
- arguments.batch_dim as u32,
+ batch_dim as u32,
encoder,
)
};
}
- if arguments.batch_dim >= 5 && self.output_dim > 1 {
- if let Some(kernel) = self.matrix_matrix.pick(arguments.batch_dim) {
- encode_kernel!(kernel, arguments.hadamard_factors);
- return Ok(());
+ if batch_dim >= 5 && self.output_dim > 1 {
+ if let Some(kernel) = self.matrix_matrix.pick(batch_dim) {
+ encode_kernel!(kernel, hadamard_factors);
+ return;
}
}
- // Matrix-vector fallback: small batch_dim, scalar output, or Big-tile-only
- // configs that don't cover this batch_dim.
match &self.matrix_vector {
- MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),
- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),
+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k),
+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, hadamard_factors),
}
-
- Ok(())
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/sampling.rs b/crates/backend-uzu/src/backends/common/kernel/sampling.rs
index 9f0c24fc4..34e78135f 100644
--- a/crates/backend-uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/sampling.rs
@@ -1,15 +1,11 @@
-use std::{
- cell::RefCell,
- mem::size_of,
- ops::{Deref, DerefMut},
-};
+use std::mem::size_of;
use thiserror::Error;
use crate::{
DataType,
backends::common::{
- Backend, Context, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
gpu_types::ArgmaxPair,
kernel::{
ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,
@@ -32,7 +28,6 @@ enum ArgmaxImplementation<B: Backend> {
TwoPass {
main_kernel: <B::Kernels as Kernels>::ArgmaxMainKernel,
final_kernel: <B::Kernels as Kernels>::ArgmaxFinalKernel,
- partial_results_buffer: RefCell<B::DenseBuffer>,
},
}
@@ -44,8 +39,6 @@ pub struct SamplingKernel<B: Backend> {
minp: <B::Kernels as Kernels>::MinPKernel,
gumbel: <B::Kernels as Kernels>::GumbelKernel,
argmax_implementation: ArgmaxImplementation<B>,
- max_batch_size: usize,
- max_vocab_size: usize,
}
#[derive(Debug, Error)]
@@ -54,27 +47,23 @@ pub enum SamplingError<B: Backend> {
BackendError(#[source] B::Error),
#[error("Function not found: {0}")]
FunctionNotFound(String),
- #[error("Batch size {0} exceeds maximum {1}")]
- BatchSizeExceeded(usize, usize),
- #[error("Vocab size {0} exceeds maximum {1}")]
- VocabSizeExceeded(usize, usize),
+ #[error("batch size must be greater than 0")]
+ EmptyBatch,
+ #[error("vocab size must be greater than 0")]
+ EmptyVocab,
}
impl<B: Backend> SamplingKernel<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
- max_batch_size: usize,
- max_vocab_size: usize,
) -> Result<Self, SamplingError<B>> {
- Self::new_with_strategy(context, data_type, max_batch_size, max_vocab_size, ArgmaxStrategy::TwoPass)
+ Self::new_with_strategy(context, data_type, ArgmaxStrategy::TwoPass)
}
pub fn new_with_strategy(
context: &B::Context,
data_type: DataType,
- max_batch_size: usize,
- max_vocab_size: usize,
argmax_strategy: ArgmaxStrategy,
) -> Result<Self, SamplingError<B>> {
let bitmask = <B::Kernels as Kernels>::BitmaskKernel::new(context, data_type, true)
@@ -105,23 +94,9 @@ impl<B: Backend> SamplingKernel<B> {
let final_kernel =
<B::Kernels as Kernels>::ArgmaxFinalKernel::new(context).map_err(SamplingError::BackendError)?;
- let block_size = 1024;
- let grain_size = 4;
-
- let elements_per_group = block_size * grain_size;
- let max_vocab_groups_per_batch = (max_vocab_size + elements_per_group - 1) / elements_per_group;
- let max_partial_results = max_batch_size * max_vocab_groups_per_batch;
-
- let partial_results_buffer = RefCell::new(
- context
- .create_buffer(max_partial_results * size_of::<ArgmaxPair>())
- .expect("Failed to create partial results buffer"),
- );
-
ArgmaxImplementation::TwoPass {
main_kernel,
final_kernel,
- partial_results_buffer,
}
},
};
@@ -134,36 +109,32 @@ impl<B: Backend> SamplingKernel<B> {
minp,
gumbel,
argmax_implementation,
- max_batch_size,
- max_vocab_size,
})
}
pub fn encode(
&self,
- mut logits_buffer: &mut B::DenseBuffer,
- seeds_buffer: &B::DenseBuffer,
- seeds_offset: usize,
- bitmask_buffer: Option<&B::DenseBuffer>,
- bitmask_offset: usize,
- sampled_tokens_buffer: &mut B::DenseBuffer,
+ logits: &mut Allocation<B>,
+ seeds: &Allocation<B>,
+ bitmask: Option<&Allocation<B>>,
+ sampled_tokens: &mut Allocation<B>,
sampling_method: SamplingMethod,
batch_size: usize,
vocab_size: usize,
encoder: &mut Encoder<B>,
) -> Result<(), SamplingError<B>> {
- if batch_size > self.max_batch_size {
- return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));
+ if batch_size == 0 {
+ return Err(SamplingError::EmptyBatch);
}
- if vocab_size > self.max_vocab_size {
- return Err(SamplingError::VocabSizeExceeded(vocab_size, self.max_vocab_size));
+ if vocab_size == 0 {
+ return Err(SamplingError::EmptyVocab);
}
- if let Some(bitmask_buffer) = bitmask_buffer {
+ if let Some(bitmask) = bitmask {
self.bitmask.encode(
- None::<&B::DenseBuffer>,
- (bitmask_buffer, bitmask_offset),
- logits_buffer.deref_mut(),
+ None::<&Allocation<B>>,
+ bitmask,
+ &mut *logits,
batch_size as u32,
vocab_size as u32,
encoder,
@@ -182,8 +153,8 @@ impl<B: Backend> SamplingKernel<B> {
&& processing_order == SamplingProcessingOrder::TemperatureThenFilters
{
self.temperature.encode(
- None::<&B::DenseBuffer>,
- logits_buffer.deref_mut(),
+ None::<&Allocation<B>>,
+ &mut *logits,
batch_size as u32,
vocab_size as u32,
temperature,
@@ -193,8 +164,8 @@ impl<B: Backend> SamplingKernel<B> {
if let Some(top_k) = top_k {
self.topk.encode(
- None::<&B::DenseBuffer>,
- logits_buffer.deref_mut(),
+ None::<&Allocation<B>>,
+ &mut *logits,
batch_size as u32,
vocab_size as u32,
top_k,
@@ -203,8 +174,8 @@ impl<B: Backend> SamplingKernel<B> {
}
if let Some(top_p) = top_p {
self.topp.encode(
- None::<&B::DenseBuffer>,
- logits_buffer.deref_mut(),
+ None::<&Allocation<B>>,
+ &mut *logits,
batch_size as u32,
vocab_size as u32,
top_p,
@@ -213,8 +184,8 @@ impl<B: Backend> SamplingKernel<B> {
}
if let Some(min_p) = min_p {
self.minp.encode(
- None::<&B::DenseBuffer>,
- logits_buffer.deref_mut(),
+ None::<&Allocation<B>>,
+ &mut *logits,
batch_size as u32,
vocab_size as u32,
min_p,
@@ -226,8 +197,8 @@ impl<B: Backend> SamplingKernel<B> {
&& processing_order == SamplingProcessingOrder::FiltersThenTemperature
{
self.temperature.encode(
- None::<&B::DenseBuffer>,
- logits_buffer.deref_mut(),
+ None::<&Allocation<B>>,
+ &mut *logits,
batch_size as u32,
vocab_size as u32,
temperature,
@@ -236,10 +207,10 @@ impl<B: Backend> SamplingKernel<B> {
}
self.gumbel.encode(
- None::<&B::DenseBuffer>,
- seeds_buffer,
- (seeds_offset / size_of::<u64>()) as u32,
- logits_buffer.deref_mut(),
+ None::<&Allocation<B>>,
+ seeds,
+ 0,
+ &mut *logits,
batch_size as u32,
vocab_size as u32,
encoder,
@@ -250,33 +221,22 @@ impl<B: Backend> SamplingKernel<B> {
ArgmaxImplementation::SinglePass {
kernel,
} => {
- kernel.encode(
- logits_buffer.deref(),
- sampled_tokens_buffer,
- batch_size as u32,
- vocab_size as u32,
- encoder,
- );
+ kernel.encode(&*logits, sampled_tokens, batch_size as u32, vocab_size as u32, encoder);
},
ArgmaxImplementation::TwoPass {
main_kernel,
final_kernel,
- partial_results_buffer,
} => {
- main_kernel.encode(
- logits_buffer.deref(),
- partial_results_buffer.borrow_mut().deref_mut(),
- batch_size as u32,
- vocab_size as u32,
- encoder,
- );
- final_kernel.encode(
- partial_results_buffer.borrow().deref(),
- sampled_tokens_buffer,
- batch_size as u32,
- vocab_size as u32,
- encoder,
- );
+ let block_size = 1024;
+ let grain_size = 4;
+ let elements_per_group = block_size * grain_size;
+ let vocab_groups_per_batch = vocab_size.div_ceil(elements_per_group);
+ let partial_results_count = batch_size * vocab_groups_per_batch;
+ let mut partial_results = encoder
+ .allocate_scratch(partial_results_count * size_of::<ArgmaxPair>())
+ .map_err(SamplingError::BackendError)?;
+ main_kernel.encode(&*logits, &mut partial_results, batch_size as u32, vocab_size as u32, encoder);
+ final_kernel.encode(&partial_results, sampled_tokens, batch_size as u32, vocab_size as u32, encoder);
},
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
index 64f473798..b3769ec4a 100644
--- a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -1,7 +1,7 @@
use crate::{
DataType,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},
},
};
@@ -13,14 +13,14 @@ pub enum SSDPrefillMode {
}
pub struct SSDPrefillArguments<'a, B: Backend> {
- pub x: &'a B::DenseBuffer,
- pub dt: &'a B::DenseBuffer, // raw dt values
- pub b: &'a B::DenseBuffer,
- pub c: &'a B::DenseBuffer,
- pub d: &'a B::DenseBuffer,
- pub z: &'a B::DenseBuffer,
- pub state: &'a mut B::DenseBuffer,
- pub y: &'a mut B::DenseBuffer,
+ pub x: &'a Allocation<B>,
+ pub dt: &'a Allocation<B>,
+ pub b: &'a Allocation<B>,
+ pub c: &'a Allocation<B>,
+ pub d: &'a Allocation<B>,
+ pub z: &'a Allocation<B>,
+ pub state: &'a mut Allocation<B>,
+ pub y: &'a mut Allocation<B>,
pub suffix_len: usize,
pub group_size: u32,
pub state_size: u32,
diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rs
index 848846f45..8dc4adada 100644
--- a/crates/backend-uzu/src/backends/common/mod.rs
+++ b/crates/backend-uzu/src/backends/common/mod.rs
@@ -1,4 +1,5 @@
mod activation_config;
+mod allocation_access;
mod allocator;
mod backend;
mod buffer;
@@ -12,6 +13,9 @@ mod hazard_tracker;
pub mod kernel;
pub use activation_config::ActivationConfig;
+pub use allocation_access::{
+ AllocationAccessError, allocation_as_bytes, allocation_as_bytes_mut, allocation_copy_from_slice, allocation_to_vec,
+};
pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};
pub use backend::Backend;
pub use buffer::{
diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
index 7d2d8abcb..f2056e35e 100644
--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
@@ -1,5 +1,4 @@
use std::{
- cell::UnsafeCell,
pin::Pin,
sync::{
atomic::{AtomicU64, Ordering},
@@ -12,8 +11,8 @@ use std::{
use crate::{
backends::{
common::{
- AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferInitial, CommandBufferPending,
+ AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,
+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
},
cpu::{Cpu, error::CpuError},
},
@@ -76,18 +75,16 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
fn encode_copy(
&mut self,
- src: &UnsafeCell<Pin<Box<[u8]>>>,
- src_range: std::ops::Range<usize>,
- dst: &mut UnsafeCell<Pin<Box<[u8]>>>,
- dst_range: std::ops::Range<usize>,
+ src: BufferRangeRef<'_, <Cpu as Backend>::DenseBuffer>,
+ dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,
) {
+ let src_range = src.range();
+ let dst_range = dst.range();
let size = src_range.end - src_range.start;
assert_eq!(size, dst_range.end - dst_range.start);
- assert!(unsafe { &*src.get() }.len() >= src_range.end);
- assert!(dst.get_mut().len() >= dst_range.end);
- let src_ptr = SendPtr(unsafe { (&*src.get()).as_ptr().add(src_range.start) });
- let dst_ptr = SendPtrMut(unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) });
+ let src_ptr = SendPtr(unsafe { (&*src.buffer().get()).as_ptr().add(src_range.start) });
+ let dst_ptr = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(dst_range.start) });
self.push_command(move || unsafe {
std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), size);
});
@@ -95,12 +92,12 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {
fn encode_fill(
&mut self,
- dst: &mut UnsafeCell<Pin<Box<[u8]>>>,
- range: std::ops::Range<usize>,
+ dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,
value: u8,
) {
+ let range = dst.range();
let size = range.end - range.start;
- let dst = SendPtrMut(dst.get_mut()[range].as_ptr() as *mut u8);
+ let dst = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(range.start) });
self.push_command(move || unsafe {
dst.as_ptr().write_bytes(value, size);
});
diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rs
index 917a679b3..2e2f96c18 100644
--- a/crates/backend-uzu/src/backends/cpu/context.rs
+++ b/crates/backend-uzu/src/backends/cpu/context.rs
@@ -37,8 +37,9 @@ impl Context for CpuContext {
fn recommended_async_batch_size(
&self,
- _model_path: &Path,
+ model_path: &Path,
) -> usize {
+ let _ = model_path;
1
}
@@ -88,8 +89,9 @@ impl Context for CpuContext {
fn start_capture(
&self,
- _trace_path: &std::path::Path,
+ trace_path: &std::path::Path,
) -> Result<(), CpuError> {
+ let _ = trace_path;
Err(CpuError::NotSupported)
}
diff --git a/crates/backend-uzu/src/backends/cpu/dense_buffer.rs b/crates/backend-uzu/src/backends/cpu/dense_buffer.rs
index e5b39b3ac..507c1ec88 100644
--- a/crates/backend-uzu/src/backends/cpu/dense_buffer.rs
+++ b/crates/backend-uzu/src/backends/cpu/dense_buffer.rs
@@ -16,8 +16,9 @@ impl Buffer for UnsafeCell<Pin<Box<[u8]>>> {
fn set_label(
&mut self,
- _label: Option<&str>,
+ label: Option<&str>,
) {
+ let _ = label;
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 33aa51c67..fe73bacc4 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -4,7 +4,7 @@ use crate::{
DataType,
backends::{
common::{
- Encoder,
+ AsBufferRangeMut, AsBufferRangeRef, Encoder,
kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
},
cpu::{Cpu, context::CpuContext},
@@ -33,28 +33,52 @@ impl MatmulKernel for MatmulCpuKernel {
fn encode(
&mut self,
- _context: &CpuContext,
arguments: MatmulArguments<Cpu>,
encoder: &mut Encoder<Cpu>,
) {
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ ab_scale,
+ c,
+ d,
+ batch_dim,
+ input_dim,
+ output_dim,
+ } = arguments;
let command_buffer = encoder.as_command_buffer_mut();
- let m = arguments.batch_dim as usize;
- let n = arguments.output_dim as usize;
- let k = arguments.input_dim as usize;
+ let m = batch_dim as usize;
+ let n = output_dim as usize;
+ let k = input_dim as usize;
let lda = k;
let ldb = k;
let ldd = n;
- let a_offset = arguments.a_offset as usize;
let data_type = self.data_type;
+ let a_buffer_range = a.as_buffer_range_ref();
+ let b_buffer_range = b.as_buffer_range_ref();
+ let d_buffer_range = d.as_buffer_range_mut();
+ let a_offset = a_buffer_range.range().start + a_offset;
+ let b_offset = b_buffer_range.range().start;
+ let d_offset = d_buffer_range.range().start;
- let a_ptr = SendPtr(unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset));
- let b_ptr = SendPtr(unsafe { &*arguments.b.get() }.as_ptr());
- let ab_scale = arguments.ab_scale;
- let d_ptr = SendPtrMut(arguments.d.get_mut().as_mut_ptr());
+ let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));
+ let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(b_offset));
+ let d_ptr =
+ SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_offset) as *mut u8 });
- let (is_accumulate, bias_ptr) = match arguments.c {
+ let (is_accumulate, bias_ptr) = match c {
MatmulArgumentC::Accumulate => (true, None),
- MatmulArgumentC::Bias(bias) => (false, Some(SendPtr(unsafe { &*bias.get() }.as_ptr()))),
+ MatmulArgumentC::Bias(bias) => {
+ let bias_buffer_range = bias.as_buffer_range_ref();
+ let bias_range = bias_buffer_range.range();
+ (
+ false,
+ Some(SendPtr(
+ unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start),
+ )),
+ )
+ },
MatmulArgumentC::None => (false, None),
};
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
index bbdde8210..b6ed115d4 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
@@ -1,5 +1,4 @@
pub mod gemm;
-pub mod gemv;
mod matmul;
pub use matmul::MatmulCpuKernel;
diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rs
index d8ac4af5d..37258d949 100644
--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs
@@ -1,16 +1,16 @@
use std::{cell::Cell, time::Duration};
use metal::{
- MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,
- MTLCommandBufferHandler, MTLCommandBufferStatus, MTLCommandEncoder, MTLComputeCommandEncoder, MTLEvent,
+ MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler,
+ MTLCommandBufferStatus, MTLCommandEncoder, MTLComputeCommandEncoder, MTLEvent,
};
use objc2::{Message, rc::Retained, runtime::ProtocolObject};
use super::Metal;
use crate::backends::{
common::{
- AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
- CommandBufferInitial, CommandBufferPending,
+ AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,
+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
},
metal::error::MetalError,
};
@@ -120,28 +120,27 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
fn encode_copy(
&mut self,
- src: &Retained<ProtocolObject<dyn MTLBuffer>>,
- src_range: std::ops::Range<usize>,
- dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,
- dst_range: std::ops::Range<usize>,
+ src: BufferRangeRef<'_, <Metal as Backend>::DenseBuffer>,
+ dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,
) {
+ let src_range = src.range();
+ let dst_range = dst.range();
let size = src_range.end - src_range.start;
assert_eq!(size, dst_range.end - dst_range.start);
- assert!(src.length() >= src_range.end && dst.length() >= dst_range.end);
- self.ensure_blit().copy_buffer_to_buffer(src, src_range.start, dst, dst_range.start, size);
+ self.ensure_blit().copy_buffer_to_buffer(src.buffer(), src_range.start, dst.buffer(), dst_range.start, size);
}
fn encode_fill(
&mut self,
- dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,
- range: std::ops::Range<usize>,
+ dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,
value: u8,
) {
- assert!(range.end > range.start && range.end <= dst.length());
+ let range = dst.range();
+ assert!(range.end > range.start);
assert!(range.start % 4 == 0 && range.end % 4 == 0);
- self.ensure_blit().fill_buffer_range_value(dst, range, value);
+ self.ensure_blit().fill_buffer_range_value(dst.buffer(), range, value);
}
fn encode_barrier(
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
index 1d31ba2cd..3350dfe46 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs
@@ -1,10 +1,7 @@
use crate::{
DataType,
backends::{
- common::{
- Context,
- kernel::matmul::{MatmulArgumentC, MatmulArguments},
- },
+ common::{Context, kernel::matmul::MatmulArgumentC},
metal::{Metal, context::MetalContext},
},
};
@@ -127,16 +124,19 @@ impl GemmSpecialization {
pub fn select(
context: &MetalContext,
data_type: DataType,
- arguments: &MatmulArguments<Metal>,
+ batch_dim: u32,
+ input_dim: u32,
+ output_dim: u32,
+ c: &MatmulArgumentC<Metal>,
) -> Self {
- let overall_work_elements = arguments.batch_dim * arguments.output_dim;
+ let overall_work_elements = batch_dim * output_dim;
let is_float32 = matches!(data_type, DataType::F32);
let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();
let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column) =
if context.is_high_performance() && overall_work_elements >= (1 << 20) {
if prefer_half_or_tf32 {
- if 2 * std::cmp::max(arguments.batch_dim, arguments.output_dim) > arguments.input_dim {
+ if 2 * std::cmp::max(batch_dim, output_dim) > input_dim {
(64, 64, 16, 2, 2)
} else {
(64, 32, 32, 2, 2)
@@ -156,9 +156,9 @@ impl GemmSpecialization {
block_depth,
simdgroups_per_row,
simdgroups_per_column,
- align_mn: (arguments.batch_dim % block_rows) == 0 && (arguments.output_dim % block_cols) == 0,
- align_k: (arguments.input_dim % block_depth) == 0,
- is_accumulate: matches!(arguments.c, MatmulArgumentC::Accumulate),
+ align_mn: (batch_dim % block_rows) == 0 && (output_dim % block_cols) == 0,
+ align_k: (input_dim % block_depth) == 0,
+ is_accumulate: matches!(c, MatmulArgumentC::Accumulate),
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index ea430a374..bdb779ca4 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -7,7 +7,7 @@ use crate::{
DataType,
backends::{
common::{
- Encoder,
+ Allocation, Encoder,
gpu_types::GemmParams,
kernel::{
TensorAddBiasKernel,
@@ -136,33 +136,40 @@ impl MatmulMetalKernel {
fn encode_gemv(
&mut self,
- context: &MetalContext,
encoder: &mut Encoder<Metal>,
arguments: MatmulArguments<Metal>,
) -> Result<(), MatmulError<Metal>> {
- let (is_accumulate, output_bias) = match arguments.c {
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ ab_scale,
+ c,
+ d,
+ batch_dim,
+ input_dim,
+ output_dim,
+ } = arguments;
+
+ let (is_accumulate, output_bias) = match c {
MatmulArgumentC::Accumulate => (true, None),
MatmulArgumentC::Bias(bias) => (false, Some(bias)),
MatmulArgumentC::None => (false, None),
};
- let specialization = gemv::GemvSpecialization::select(
- arguments.input_dim,
- arguments.output_dim,
- is_accumulate,
- output_bias.is_some(),
- );
+ let specialization =
+ gemv::GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());
- self.get_or_create_gemv(context, specialization)?.encode(
- arguments.b,
- (arguments.a, arguments.a_offset as usize),
+ self.get_or_create_gemv(encoder.context(), specialization)?.encode(
+ b,
+ (a, a_offset),
output_bias,
- arguments.d,
- arguments.input_dim,
- arguments.output_dim,
- arguments.input_dim,
- arguments.ab_scale,
- arguments.batch_dim,
+ d,
+ input_dim,
+ output_dim,
+ input_dim,
+ ab_scale,
+ batch_dim,
specialization.output_rows_per_threadgroup(),
encoder,
);
@@ -172,51 +179,56 @@ impl MatmulMetalKernel {
fn encode_gemm(
&mut self,
- context: &MetalContext,
encoder: &mut Encoder<Metal>,
arguments: MatmulArguments<Metal>,
) -> Result<(), MatmulError<Metal>> {
- let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);
-
- let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);
- let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ ab_scale,
+ c,
+ d,
+ batch_dim,
+ input_dim,
+ output_dim,
+ } = arguments;
+
+ let specialization =
+ gemm::GemmSpecialization::select(encoder.context(), self.data_type, batch_dim, input_dim, output_dim, &c);
+
+ let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);
+ let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);
let params = GemmParams {
- M: arguments.batch_dim,
- N: arguments.output_dim,
- K: arguments.input_dim,
- leading_dimension_a: arguments.input_dim,
- leading_dimension_b: arguments.input_dim,
- leading_dimension_d: arguments.output_dim,
+ M: batch_dim,
+ N: output_dim,
+ K: input_dim,
+ leading_dimension_a: input_dim,
+ leading_dimension_b: input_dim,
+ leading_dimension_d: output_dim,
threadgroups_per_row,
threadgroups_per_column,
swizzle_log: 0,
- aligned_inner_iterations: arguments.input_dim / specialization.block_depth,
+ aligned_inner_iterations: input_dim / specialization.block_depth,
use_morton: false,
};
- let kernel = self.get_or_create_gemm(context, specialization)?;
+ let kernel = self.get_or_create_gemm(encoder.context(), specialization)?;
kernel.encode(
- (arguments.a, arguments.a_offset as usize),
- arguments.b,
- &mut *arguments.d,
+ (a, a_offset),
+ b,
+ &mut *d,
std::slice::from_ref(&params),
threadgroups_per_row,
threadgroups_per_column,
- arguments.ab_scale,
+ ab_scale,
encoder,
);
- if let MatmulArgumentC::Bias(bias) = arguments.c {
- self.bias_add.encode(
- None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,
- bias,
- arguments.d,
- arguments.output_dim,
- arguments.batch_dim * arguments.output_dim,
- encoder,
- );
+ if let MatmulArgumentC::Bias(bias) = c {
+ self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);
}
Ok(())
@@ -224,21 +236,26 @@ impl MatmulMetalKernel {
fn encode_gemm_mpp(
&mut self,
- context: &MetalContext,
encoder: &mut Encoder<Metal>,
arguments: MatmulArguments<Metal>,
) -> Result<(), MatmulError<Metal>> {
- let is_accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);
- let specialization = gemm_mpp::GemmMppSpecialization::select(
- arguments.batch_dim,
- arguments.output_dim,
- arguments.input_dim,
- is_accumulate,
- arguments.ab_scale != 1.0,
- );
-
- let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);
- let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ ab_scale,
+ c,
+ d,
+ batch_dim,
+ input_dim,
+ output_dim,
+ } = arguments;
+ let is_accumulate = matches!(c, MatmulArgumentC::Accumulate);
+ let specialization =
+ gemm_mpp::GemmMppSpecialization::select(batch_dim, output_dim, input_dim, is_accumulate, ab_scale != 1.0);
+
+ let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);
+ let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);
let max_threadgroups_per_dim = threadgroups_per_row.max(threadgroups_per_column);
let min_threadgroups_per_dim = threadgroups_per_row.min(threadgroups_per_column);
@@ -250,12 +267,12 @@ impl MatmulMetalKernel {
let use_morton = min_threadgroups_per_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);
let params = GemmParams {
- M: arguments.batch_dim,
- N: arguments.output_dim,
- K: arguments.input_dim,
- leading_dimension_a: arguments.input_dim,
- leading_dimension_b: arguments.input_dim,
- leading_dimension_d: arguments.output_dim,
+ M: batch_dim,
+ N: output_dim,
+ K: input_dim,
+ leading_dimension_a: input_dim,
+ leading_dimension_b: input_dim,
+ leading_dimension_d: output_dim,
threadgroups_per_row,
threadgroups_per_column,
swizzle_log: 0,
@@ -269,27 +286,20 @@ impl MatmulMetalKernel {
(threadgroups_per_row, threadgroups_per_column)
};
- let kernel = self.get_or_create_gemm_mpp(context, specialization)?;
+ let kernel = self.get_or_create_gemm_mpp(encoder.context(), specialization)?;
kernel.encode(
- (arguments.a, arguments.a_offset as usize),
- arguments.b,
- &mut *arguments.d,
+ (a, a_offset),
+ b,
+ &mut *d,
std::slice::from_ref(&params),
group_count_x,
group_count_y,
- arguments.ab_scale,
+ ab_scale,
encoder,
);
- if let MatmulArgumentC::Bias(bias) = arguments.c {
- self.bias_add.encode(
- None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,
- bias,
- arguments.d,
- arguments.output_dim,
- arguments.batch_dim * arguments.output_dim,
- encoder,
- );
+ if let MatmulArgumentC::Bias(bias) = c {
+ self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);
}
Ok(())
@@ -308,18 +318,15 @@ pub enum MatmulDispatchPath {
impl MatmulMetalKernel {
pub fn encode_with_path(
&mut self,
- context: &MetalContext,
arguments: MatmulArguments<Metal>,
encoder: &mut Encoder<Metal>,
path: MatmulDispatchPath,
) {
match path {
- MatmulDispatchPath::Auto => self.encode(context, arguments, encoder),
- MatmulDispatchPath::Gemv => self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV"),
- MatmulDispatchPath::Gemm => self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM"),
- MatmulDispatchPath::GemmMpp => {
- self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP")
- },
+ MatmulDispatchPath::Auto => self.encode(arguments, encoder),
+ MatmulDispatchPath::Gemv => self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV"),
+ MatmulDispatchPath::Gemm => self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM"),
+ MatmulDispatchPath::GemmMpp => self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP"),
}
}
}
@@ -362,16 +369,15 @@ impl MatmulKernel for MatmulMetalKernel {
fn encode(
&mut self,
- context: &MetalContext,
arguments: MatmulArguments<Metal>,
encoder: &mut Encoder<Metal>,
) {
if arguments.batch_dim <= max_gemv_batch_threshold() {
- self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");
- } else if self.is_mpp_eligible(context) {
- self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP kernel");
+ self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV kernel");
+ } else if self.is_mpp_eligible(encoder.context()) {
+ self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP kernel");
} else {
- self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");
+ self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index 5f375544b..d4f1eb964 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -217,4 +217,4 @@ PUBLIC KERNEL(ShortConvTrie)(
dst_state[tap_count - 1] = static_cast<T>(x);
}
}
-}
\ No newline at end of file
+}
diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rs
index ddf9a6997..7683b416a 100644
--- a/crates/backend-uzu/src/classifier/classifier.rs
+++ b/crates/backend-uzu/src/classifier/classifier.rs
@@ -1,16 +1,14 @@
-#[cfg(feature = "tracing")]
-use std::{cell::RefCell, rc::Rc};
use std::{collections::HashMap, path::Path, time::Instant};
#[cfg(feature = "tracing")]
use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
- DataType,
- backends::common::{Backend, Encoder},
+ DataType, allocation_to_vec,
+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
config::ModelMetadata,
- encodable_block::EncodingParameters,
- forward_pass::state::{ArrayId, ForwardPassState},
+ encodable_block::{EncodingParameters, LayerArguments},
+ forward_pass::token_inputs::TokenInputs,
session::types::Error,
};
@@ -34,10 +32,6 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {
) -> Result<ClassificationOutput, Error> {
let run_start = Instant::now();
- #[cfg(feature = "tracing")]
- let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;
-
- #[cfg(not(feature = "tracing"))]
let logits = self.forward_pass(&token_ids, &token_positions)?;
let forward_duration = run_start.elapsed().as_secs_f64();
@@ -86,8 +80,16 @@ impl<B: Backend> Classifier<B> {
&mut self,
token_ids: &[u64],
token_positions: &[usize],
- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {
- self.forward_pass(token_ids, token_positions)
+ ) -> Result<(Box<[f32]>, ActivationTrace<B>), Error> {
+ let num_labels = self.context.model_config.model_config.num_labels;
+ let mut traces = ActivationTrace::new_classifier(
+ self.context.context.as_ref(),
+ &self.context.model_shape,
+ token_ids.len(),
+ num_labels,
+ );
+ let logits = self.forward_pass_impl(token_ids, token_positions, Some(&mut traces))?;
+ Ok((logits, traces))
}
#[cfg(feature = "tracing")]
@@ -95,51 +97,110 @@ impl<B: Backend> Classifier<B> {
&mut self,
token_ids: &[u64],
token_positions: &[usize],
- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {
- let num_labels = self.context.model_config.model_config.num_labels;
- let mut state = ForwardPassState::new_classifier(
- self.context.context.clone(),
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.shared_buffers.clone(),
- token_ids,
- token_positions,
- num_labels,
- );
+ ) -> Result<Box<[f32]>, Error> {
+ self.forward_pass_impl(token_ids, token_positions, None)
+ }
+ fn forward_pass_impl(
+ &mut self,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ #[cfg(feature = "tracing")] trace: Option<&mut ActivationTrace<B>>,
+ ) -> Result<Box<[f32]>, Error> {
+ let batch_dim = token_ids.len();
+ let token_inputs = TokenInputs::new_classifier(self.context.context.as_ref(), token_ids, token_positions);
let encoding_params = EncodingParameters::new();
+ #[cfg(feature = "tracing")]
+ let mut trace = trace;
let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
.map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
- self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ let mut main = self
+ .context
+ .embed
+ .encode_lookup(token_inputs.token_ids(), batch_dim, &mut encoder)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ main = self
+ .context
+ .embedding_norm
+ .encode(&main, 0, batch_dim, &mut encoder)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
- {
- let traces = state.traces().clone();
- state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().embedding_norm().clone());
+ if let Some(trace) = trace.as_deref_mut() {
+ let embedding_norm = trace.embedding_norm_mut().allocation_mut();
+ encoder.encode_copy(&main, .., embedding_norm, ..);
}
- for layer in self.context.layers.iter() {
- layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ let mut shortcut = encoder
+ .allocate_scratch(main.as_buffer_range_ref().range().len())
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ for (layer_index, layer) in self.context.layers.iter().enumerate() {
+ let rope_type = layer.rope_type();
+ main = layer
+ .encode(
+ LayerArguments {
+ batch_dim,
+ token_positions: token_inputs.token_positions(),
+ token_parents: token_inputs.token_parents(),
+ token_subtrie_ranges: None,
+ attention_sinks: self.context.shared_buffers.attention_sinks(layer_index),
+ rope_cosines: self.context.shared_buffers.rope_cosines(rope_type),
+ rope_sines: self.context.shared_buffers.rope_sines(rope_type),
+ rope_max_sequence_length: self.context.model_shape.context_length(),
+ rope_dim: self.context.model_shape.rope_dim(),
+ sampling_start: 0,
+ sampling_length: batch_dim,
+ cache_layer: None,
+ #[cfg(feature = "tracing")]
+ trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer_index]),
+ },
+ &encoding_params,
+ main,
+ &mut shortcut,
+ &mut encoder,
+ )
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
- self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ main = self
+ .context
+ .output_norm
+ .encode(&main, 0, batch_dim, &mut encoder)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
- {
- let traces = state.traces().clone();
- state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().output_norm.clone());
+ if let Some(trace) = trace.as_deref_mut() {
+ encoder.encode_copy(&main, .., trace.output_norm.allocation_mut(), ..);
+ }
+ let pooling = self
+ .context
+ .pooling
+ .encode(batch_dim, &main, &mut encoder)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ #[cfg(feature = "tracing")]
+ if let Some(trace) = trace.as_deref_mut() {
+ let output_pooling = trace.output_pooling_mut().allocation_mut();
+ encoder.encode_copy(&pooling, .., output_pooling, ..);
+ }
+ let logits =
+ self.context.prediction_head.encode(pooling, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ #[cfg(feature = "tracing")]
+ if let Some(trace) = trace.as_deref_mut() {
+ encoder.encode_copy(&logits, .., trace.logits.allocation_mut(), ..);
}
- self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ let completed = encoder
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
- let logits = self.copy_logits_from_state(&state)?;
+ let logits_result = self.copy_logits_from_allocation(&logits);
+ drop(logits);
+ drop(main);
+ drop(shortcut);
+ drop(completed);
- let traces = state.traces().clone();
- Ok((logits, traces))
+ logits_result
}
#[cfg(not(feature = "tracing"))]
@@ -148,49 +209,24 @@ impl<B: Backend> Classifier<B> {
token_ids: &[u64],
token_positions: &[usize],
) -> Result<Box<[f32]>, Error> {
- let num_labels = self.context.model_config.model_config.num_labels;
- let mut state = ForwardPassState::new_classifier(
- self.context.context.clone(),
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.shared_buffers.clone(),
- token_ids,
- token_positions,
- num_labels,
- );
-
- let encoding_params = EncodingParameters::new();
-
- let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
-
- self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- for layer in self.context.layers.iter() {
- layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- }
- self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
-
- let logits = self.copy_logits_from_state(&state)?;
-
- Ok(logits)
+ self.forward_pass_impl(token_ids, token_positions)
}
- fn copy_logits_from_state(
+ fn copy_logits_from_allocation(
&self,
- state: &ForwardPassState<B>,
+ logits: &Allocation<B>,
) -> Result<Box<[f32]>, Error> {
- let num_labels = self.context.model_config.model_config.num_labels;
- let logits = state.array(ArrayId::ClassifierPredictionHeadLogits).view(&[num_labels]);
-
- match logits.data_type() {
- DataType::F32 => Ok(logits.as_slice::<f32>().into()),
- DataType::F16 => Ok(logits.as_slice::<half::f16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),
- DataType::BF16 => Ok(logits.as_slice::<half::bf16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),
+ let logits_data_type: DataType =
+ self.context.model_config.model_config.prediction_head_config.readout_config.activation_precision().into();
+
+ match logits_data_type {
+ DataType::F32 => Ok(allocation_to_vec::<B, f32>(logits).into()),
+ DataType::F16 => {
+ Ok(allocation_to_vec::<B, half::f16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())
+ },
+ DataType::BF16 => {
+ Ok(allocation_to_vec::<B, half::bf16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())
+ },
_ => Err(Error::UnableToDecodeText),
}
}
diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rs
index 2fe163e94..b526b8884 100644
--- a/crates/backend-uzu/src/classifier/classifier_context.rs
+++ b/crates/backend-uzu/src/classifier/classifier_context.rs
@@ -1,17 +1,14 @@
-use std::{cell::RefCell, fs::File, path::Path, rc::Rc};
+use std::{fs::File, path::Path, rc::Rc};
use crate::{
DataType,
backends::common::{Backend, Context},
classifier::ClassifierError,
config::{ClassifierModelConfig, ModelMetadata},
- encodable_block::{
- Activation, ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, Rope,
- },
+ encodable_block::{ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, Rope},
forward_pass::{
model_shape::ModelShape,
- scratch_buffers::ScratchBuffers,
- state::{ArrayId, RopeType, SharedBuffers},
+ state::{RopeType, SharedBuffers},
},
parameters::ParameterLoader,
session::types::Error,
@@ -20,8 +17,7 @@ use crate::{
pub struct ClassifierContext<B: Backend> {
pub context: Rc<B::Context>,
- pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- pub scratch_buffers: ScratchBuffers<B>,
+ pub shared_buffers: Rc<SharedBuffers<B>>,
pub model_config: ClassifierModelConfig,
pub model_shape: ModelShape,
@@ -56,20 +52,18 @@ impl<B: Backend> ClassifierContext<B> {
let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let root_loader_view = loader.tree();
- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
let transformer_tree = root_loader_view
.subtree("transformer")
.map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("transformer".to_string())))?;
- {
- let mut shared_bufs = shared_buffers.borrow_mut();
- if let Some(global_rope) = &mut shared_bufs.global_rope {
- global_rope.update_data(&transformer_tree, "global_rope");
- }
- if let Some(local_rope) = &mut shared_bufs.local_rope {
- local_rope.update_data(&transformer_tree, "local_rope");
- }
+ if let Some(global_rope) = &mut shared_buffers.global_rope {
+ global_rope.update_data(&transformer_tree, "global_rope");
+ }
+ if let Some(local_rope) = &mut shared_buffers.local_rope {
+ local_rope.update_data(&transformer_tree, "local_rope");
}
+ let shared_buffers = Rc::new(shared_buffers);
let data_type = decoder_config
.layer_config
@@ -154,15 +148,10 @@ impl<B: Backend> ClassifierContext<B> {
context.as_ref(),
data_type,
classifier_model_config.model_config.transformer_config.output_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&output_norm_tree,
)
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;
- let context_length = classifier_model_config.model_config.context_length;
- let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length);
-
let embedding_norm_tree = root_loader_view
.subtree("embedding_norm")
.map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("embedding_norm".to_string())))?;
@@ -170,8 +159,6 @@ impl<B: Backend> ClassifierContext<B> {
context.as_ref(),
data_type,
classifier_model_config.model_config.embedding_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&embedding_norm_tree,
)
.map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("embedding norm: {:?}", e))))?;
@@ -190,24 +177,13 @@ impl<B: Backend> ClassifierContext<B> {
})?;
let prediction_head_dense = <dyn Linear<B>>::new::<1>(
&prediction_head_config.dense_config,
- prediction_head_config.use_dense_bias,
model_dim,
[model_dim],
context.as_ref(),
&prediction_head_dense_tree,
- ArrayId::ClassifierPooling,
- ArrayId::ClassifierPredictionHeadDense,
- );
-
- let prediction_head_activation = Activation::<B>::new(
- &context,
- prediction_head_data_type,
- prediction_head_config.activation.clone(),
- ArrayId::ClassifierPredictionHeadDense,
- ArrayId::ClassifierPredictionHeadDense,
)
.map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))
})?;
let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {
@@ -217,8 +193,6 @@ impl<B: Backend> ClassifierContext<B> {
context.as_ref(),
prediction_head_data_type,
prediction_head_config.normalization_config.clone(),
- ArrayId::ClassifierPredictionHeadDense,
- ArrayId::ClassifierPredictionHeadNorm,
&prediction_head_norm_tree,
)
.map_err(|e| {
@@ -230,13 +204,10 @@ impl<B: Backend> ClassifierContext<B> {
})?;
let prediction_head_final_linear = <dyn Linear<B>>::new::<1>(
&prediction_head_config.readout_config,
- true,
model_dim,
[num_labels],
context.as_ref(),
&prediction_head_readout_tree,
- ArrayId::ClassifierPredictionHeadNorm,
- ArrayId::ClassifierPredictionHeadLogits,
)
.map_err(|e| {
Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head readout: {:?}", e)))
@@ -254,19 +225,21 @@ impl<B: Backend> ClassifierContext<B> {
})?;
let prediction_head = ClassifierPredictionHead::new(
- prediction_head_dense.map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))
- })?,
- prediction_head_activation,
+ context.as_ref(),
+ prediction_head_dense,
+ prediction_head_config.activation.clone(),
+ prediction_head_data_type,
prediction_head_norm,
prediction_head_final_linear,
- num_labels,
- );
+ model_dim,
+ )
+ .map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))
+ })?;
Ok(Self {
context,
shared_buffers,
- scratch_buffers,
model_config: classifier_model_config.clone(),
model_shape,
embed,
diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rs
index f674948da..acc0aaa7a 100644
--- a/crates/backend-uzu/src/config/mod.rs
+++ b/crates/backend-uzu/src/config/mod.rs
@@ -29,7 +29,6 @@ pub use classifier_model::*;
pub use common::*;
pub use decoder::*;
pub use decoder_layer::*;
-#[allow(unused_imports)]
pub use delta_net::*;
pub use embedding::*;
pub use error::*;
diff --git a/crates/backend-uzu/src/encodable_block/activation.rs b/crates/backend-uzu/src/encodable_block/activation.rs
deleted file mode 100644
index 55598deec..000000000
--- a/crates/backend-uzu/src/encodable_block/activation.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-//! Activation encodable.
-
-use std::ops::DerefMut;
-
-use crate::{
- DataType,
- backends::common::{
- ActivationConfig, Backend, Encoder,
- gpu_types::ActivationType,
- kernel::{ActivationKernel, Kernels},
- },
- forward_pass::state::{ArrayId, ForwardPassState},
-};
-
-pub struct Activation<B: Backend> {
- kernel: <B::Kernels as Kernels>::ActivationKernel,
- activation: ActivationConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
-}
-
-impl<B: Backend> Activation<B> {
- pub fn new(
- context: &B::Context,
- data_type: DataType,
- config: ActivationConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- ) -> Result<Self, B::Error> {
- let kernel =
- <B::Kernels as Kernels>::ActivationKernel::new(context, data_type, input_array_id == output_array_id)?;
- Ok(Self {
- kernel,
- activation: config,
- input_array_id,
- output_array_id,
- })
- }
-
- pub fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let input_array = state.array(self.input_array_id);
- let output_array = state.array(self.output_array_id);
-
- let n = input_array.shape().iter().product::<usize>();
- if self.activation.act_type() == ActivationType::IDENTITY {
- panic!("Identity activation is not supported for kernel");
- }
-
- let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());
- let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());
- self.kernel.encode(
- input_buffer_borrow.as_deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- n as u32,
- self.activation.act_type(),
- encoder,
- );
- Ok(())
- }
-}
diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rs
index 411a3bfd0..a402b1099 100644
--- a/crates/backend-uzu/src/encodable_block/attention.rs
+++ b/crates/backend-uzu/src/encodable_block/attention.rs
@@ -1,16 +1,14 @@
//! Attention kernel encodable.
-use std::{
- collections::HashMap,
- ops::{Deref, DerefMut},
-};
+use std::collections::HashMap;
use itertools::iproduct;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
gpu_types::ring::RingParams,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,
@@ -18,11 +16,7 @@ use crate::{
attention::{AttentionGemmArguments, AttentionGemmBlock},
},
},
- encodable_block::EncodingParameters,
- forward_pass::{
- kv_cache_layer::KVCacheLayerState,
- state::{ArrayId, ForwardPassState},
- },
+ forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
};
fn env_gemm_attention_enabled() -> bool {
@@ -49,18 +43,23 @@ pub struct Attention<B: Backend> {
update_kv_cache_inplace_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,
gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,
gemm_block: AttentionGemmBlock<B>,
- layer_index: usize,
+ data_type: DataType,
attention_scale: Option<f32>,
- has_sinks: bool,
is_causal: bool,
sliding_window_size: Option<usize>,
}
+pub struct AttentionArguments<'a, B: Backend> {
+ pub projection_step: usize,
+ pub token_subtrie_ranges: Option<&'a Allocation<B>>,
+ pub attention_sinks: Option<&'a Allocation<B>>,
+ pub kv_cache_layer: Option<&'a mut KVCacheLayer<B>>,
+}
+
impl<B: Backend> Attention<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
- layer_index: usize,
attention_scale: Option<f32>,
has_sinks: bool,
is_causal: bool,
@@ -125,9 +124,8 @@ impl<B: Backend> Attention<B> {
update_kv_cache_inplace_kernel,
gate_kernel,
gemm_block,
- layer_index,
+ data_type,
attention_scale,
- has_sinks,
is_causal,
sliding_window_size,
})
@@ -164,55 +162,40 @@ impl<B: Backend> Attention<B> {
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters,
+ mut args: AttentionArguments<B>,
+ qkv: &Allocation<B>,
+ queries: &Allocation<B>,
+ mut rotated_keys: Allocation<B>,
+ gate: Option<&Allocation<B>>,
+ suffix_length: usize,
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let qkv_array = state.array(ArrayId::QKV);
- let queries_array = state.array(ArrayId::RotatedQueries);
- let rotated_keys_array = state.array(ArrayId::RotatedKeys);
- let attention_output_array = state.array(ArrayId::AttentionOutput);
- let partials_array = state.array(ArrayId::AttentionPartials);
- let sums_array = state.array(ArrayId::AttentionSums);
- let maxs_array = state.array(ArrayId::AttentionMaxs);
-
- let suffix_length = qkv_array.shape()[0];
- let num_heads = queries_array.shape()[0];
- let head_dim = queries_array.shape()[2];
- let num_groups = rotated_keys_array.shape()[0];
- let max_sequence_length = if state.cache_layers().is_some() {
- state.array(ArrayId::Keys(self.layer_index)).shape()[1]
- } else {
- // For classifiers without KV cache, max_sequence_length is just suffix_length
- suffix_length
- };
-
- let is_trie = state.token_subtrie_ranges.is_some();
-
- let (segment_prefix_length, ring_params) = if let Some(cache_layers) = state.cache_layers() {
- let projection_step = parameters.projection_step.unwrap_or(0);
- let cache = cache_layers.borrow();
- let layer = cache.data[self.layer_index]
- .as_transformer()
- .expect("Attention kernel expects transformer layer state");
- let ring_params = match layer.state {
- KVCacheLayerState::Windowed {
- ring_offset,
- ring_length,
- window_length,
- } => {
- let overflow = (ring_length + projection_step).saturating_sub(window_length);
- Some(RingParams {
- ring_offset: ((ring_offset + overflow) % window_length) as u32,
- ring_length: (ring_length + projection_step).min(window_length) as u32,
- })
- },
- _ => None,
+ ) -> Result<Allocation<B>, B::Error> {
+ let is_trie = args.token_subtrie_ranges.is_some();
+
+ let (max_sequence_length, segment_prefix_length, ring_params) =
+ if let Some(layer) = args.kv_cache_layer.as_deref() {
+ let max_sequence_length = layer.shape[0];
+ let ring_params = match layer.state {
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length,
+ } => {
+ let overflow = (ring_length + args.projection_step).saturating_sub(window_length);
+ Some(RingParams {
+ ring_offset: ((ring_offset + overflow) % window_length) as u32,
+ ring_length: (ring_length + args.projection_step).min(window_length) as u32,
+ })
+ },
+ _ => None,
+ };
+ (max_sequence_length, layer.projected_segment_prefix_length(args.projection_step), ring_params)
+ } else {
+ (suffix_length, 0, None)
};
- (layer.projected_segment_prefix_length(projection_step), ring_params)
- } else {
- (0, None)
- };
let is_kv_cache_ring = ring_params.is_some();
@@ -231,99 +214,31 @@ impl<B: Backend> Attention<B> {
let variant =
self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, is_trie, is_kv_cache_ring);
- let gate_array = self.gate_kernel.as_ref().map(|_| state.array(ArrayId::Gate));
-
- let sinks_array = self.has_sinks.then(|| state.array(ArrayId::AttentionSinks(self.layer_index)));
-
- let rotated_keys_buf_rc = rotated_keys_array.buffer();
- let mut rotated_keys_buf_borrow = rotated_keys_buf_rc.borrow_mut();
-
- let qkv_buf_rc = qkv_array.buffer();
- let qkv_buf_borrow = qkv_buf_rc.borrow();
+ let sinks_allocation = args.attention_sinks;
+ let trie_allocation = args.token_subtrie_ranges;
// Get KV cache buffers only if KV cache exists (LLM mode)
- let has_kv_cache = state.cache_layers().is_some();
-
- let key_cache_array = has_kv_cache.then(|| state.array(ArrayId::Keys(self.layer_index)));
- let key_cache_buf_rc = key_cache_array.as_ref().map(|a| a.buffer());
- let mut key_cache_buf_borrow = key_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());
-
- let value_cache_array = has_kv_cache.then(|| state.array(ArrayId::Values(self.layer_index)));
- let value_cache_buf_rc = value_cache_array.as_ref().map(|a| a.buffer());
- let mut value_cache_buf_borrow = value_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());
-
- let extracted_values_array = (!has_kv_cache).then(|| state.array(ArrayId::ExtractedValues));
- let extracted_values_buf_rc = extracted_values_array.as_ref().map(|a| a.buffer());
- let mut extracted_values_buf_borrow = extracted_values_buf_rc.as_ref().map(|rc| rc.borrow_mut());
-
- // For classifiers (no KV cache): extract values from QKV into a dedicated extracted_values buffer.
- if !has_kv_cache {
- self.update_kv_cache_inplace_kernel.encode(
- None::<&B::DenseBuffer>,
- qkv_buf_borrow.deref(),
- // keys already in desired layout; harmless overwrite
- rotated_keys_buf_borrow.deref_mut(),
- extracted_values_buf_borrow.as_mut().unwrap().deref_mut(),
- num_groups as u32,
- num_heads as u32,
- head_dim as u32,
- suffix_length as u32,
- 0u32,
- max_sequence_length as u32,
- encoder,
- );
- }
-
- let queries_buf_rc = queries_array.buffer();
- let queries_buf_borrow = queries_buf_rc.borrow();
-
- let attention_output_buf_rc = attention_output_array.buffer();
- let mut attention_output_buf_borrow = attention_output_buf_rc.borrow_mut();
-
- let trie_buf_rc = state.token_subtrie_ranges.as_ref().map(|a| a.buffer());
- let trie_buf_borrow = trie_buf_rc.as_ref().map(|rc| rc.borrow());
- let trie_buffer: Option<&B::DenseBuffer> = trie_buf_borrow.as_ref().map(|b| b.deref());
-
- let partials_buf_rc = partials_array.buffer();
- let mut partials_buf_borrow = partials_buf_rc.borrow_mut();
-
- let sums_buf_rc = sums_array.buffer();
- let mut sums_buf_borrow = sums_buf_rc.borrow_mut();
-
- let maxs_buf_rc = maxs_array.buffer();
- let mut maxs_buf_borrow = maxs_buf_rc.borrow_mut();
-
- let sinks_buf_rc = sinks_array.as_ref().map(|b| b.buffer());
- let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());
- let sinks_buffer: Option<&B::DenseBuffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());
-
- // Only update KV cache for LLM mode (not for classifiers)
- if has_kv_cache {
- self.update_kv_cache_kernel.encode(
- Some(rotated_keys_buf_borrow.deref()),
- qkv_buf_borrow.deref(),
- key_cache_buf_borrow.as_mut().unwrap().deref_mut(),
- value_cache_buf_borrow.as_mut().unwrap().deref_mut(),
- num_groups as u32,
- num_heads as u32,
- head_dim as u32,
- suffix_length as u32,
- segment_prefix_length as u32,
- max_sequence_length as u32,
- encoder,
- );
- }
-
- let key_cache_buffer: &B::DenseBuffer = if has_kv_cache {
- key_cache_buf_borrow.as_ref().unwrap().deref()
- } else {
- rotated_keys_buf_borrow.deref()
- };
- let value_cache_buffer: &B::DenseBuffer = if has_kv_cache {
- value_cache_buf_borrow.as_ref().unwrap().deref()
- } else {
- extracted_values_buf_borrow.as_ref().unwrap().deref()
- };
+ let has_kv_cache = args.kv_cache_layer.is_some();
+ let extracted_values = (!has_kv_cache)
+ .then(|| {
+ let mut values =
+ encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;
+ self.update_kv_cache_inplace_kernel.encode(
+ None::<&Allocation<B>>,
+ qkv,
+ &mut rotated_keys,
+ &mut values,
+ num_groups as u32,
+ num_heads as u32,
+ head_dim as u32,
+ suffix_length as u32,
+ 0u32,
+ max_sequence_length as u32,
+ encoder,
+ );
+ Ok(values)
+ })
+ .transpose()?;
// KV cache layout: [max_sequence_length, num_groups, head_dim] (token-major)
// For classifier mode (no KV cache) keys/values still come in group-major layout
@@ -345,15 +260,96 @@ impl<B: Backend> Attention<B> {
is_kv_cache_ring,
};
+ let (keys, values) = if let Some(layer) = args.kv_cache_layer.as_deref_mut() {
+ self.update_kv_cache_kernel.encode(
+ Some(&rotated_keys),
+ qkv,
+ &mut layer.keys,
+ &mut layer.values,
+ num_groups as u32,
+ num_heads as u32,
+ head_dim as u32,
+ suffix_length as u32,
+ segment_prefix_length as u32,
+ max_sequence_length as u32,
+ encoder,
+ );
+ (&layer.keys, &layer.values)
+ } else {
+ (&rotated_keys, extracted_values.as_ref().expect("Missing extracted values for classifier attention"))
+ };
+
+ let mut attention_output = self.encode_attention_variant(
+ variant,
+ &kernel_key,
+ queries,
+ keys,
+ values,
+ trie_allocation,
+ sinks_allocation,
+ gqa_factor,
+ sequence_length,
+ k_head_stride,
+ k_seq_stride,
+ v_head_stride,
+ v_seq_stride,
+ ring_params,
+ scale,
+ num_heads,
+ num_groups,
+ suffix_length,
+ segment_prefix_length,
+ max_sequence_length,
+ head_dim,
+ encoder,
+ )?;
+
+ if let Some(gate_kernel) = &self.gate_kernel {
+ let total_elements = (suffix_length * num_heads * head_dim) as u32;
+ let gate = gate.expect("Gate allocation not initialized");
+ gate_kernel.encode(gate, &mut attention_output, total_elements, encoder);
+ }
+
+ Ok(attention_output)
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ fn encode_attention_variant(
+ &self,
+ variant: KernelVariant,
+ kernel_key: &KernelKey,
+ queries: &Allocation<B>,
+ keys: &Allocation<B>,
+ values: &Allocation<B>,
+ trie_allocation: Option<&Allocation<B>>,
+ sinks_allocation: Option<&Allocation<B>>,
+ gqa_factor: usize,
+ sequence_length: usize,
+ k_head_stride: u32,
+ k_seq_stride: u32,
+ v_head_stride: u32,
+ v_seq_stride: u32,
+ ring_params: Option<RingParams>,
+ scale: f32,
+ num_heads: usize,
+ num_groups: usize,
+ suffix_length: usize,
+ segment_prefix_length: usize,
+ max_sequence_length: usize,
+ head_dim: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut attention_output =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads * head_dim], self.data_type))?;
match variant {
KernelVariant::Gemm => {
let args = AttentionGemmArguments {
- queries_buffer: queries_buf_borrow.deref(),
- keys_buffer: key_cache_buffer,
- values_buffer: value_cache_buffer,
- output_buffer: attention_output_buf_borrow.deref_mut(),
- trie_buffer,
- sinks_buffer,
+ queries,
+ keys,
+ values,
+ output: &mut attention_output,
+ trie: trie_allocation,
+ sinks: sinks_allocation,
num_heads,
num_groups,
suffix_length,
@@ -370,18 +366,18 @@ impl<B: Backend> Attention<B> {
v_head_stride: v_head_stride as u64,
v_seq_stride: v_seq_stride as u64,
};
- self.gemm_block.encode(state.context(), encoder, args).expect("Failed to encode AttentionGemmBlock");
+ self.gemm_block.encode(encoder, args)?;
},
KernelVariant::SinglePass => {
- let kernel = match self.single_pass_kernels.get(&kernel_key) {
- Some(k) => k,
- None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),
- };
+ let kernel = self
+ .single_pass_kernels
+ .get(kernel_key)
+ .unwrap_or_else(|| panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key));
kernel.encode(
- queries_buf_borrow.deref(),
- key_cache_buffer,
- value_cache_buffer,
- attention_output_buf_borrow.deref_mut(),
+ queries,
+ keys,
+ values,
+ &mut attention_output,
gqa_factor as u32,
sequence_length as u32,
k_head_stride,
@@ -390,30 +386,36 @@ impl<B: Backend> Attention<B> {
v_seq_stride,
ring_params,
scale,
- trie_buffer,
+ trie_allocation,
self.sliding_window_size.map(|s| s as u32),
- sinks_buffer,
+ sinks_allocation,
num_heads as u32,
suffix_length as u32,
encoder,
- )
+ );
},
KernelVariant::TwoPass => {
- let kernel_pass1 = match self.two_pass_1_kernels.get(&kernel_key) {
- Some(k) => k,
- None => panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key),
- };
- let kernel_pass2 = match self.two_pass_2_kernels.get(&(head_dim as u32)) {
- Some(k) => k,
- None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),
- };
+ let kernel_pass1 = self
+ .two_pass_1_kernels
+ .get(kernel_key)
+ .unwrap_or_else(|| panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key));
+ let kernel_pass2 = self
+ .two_pass_2_kernels
+ .get(&(head_dim as u32))
+ .unwrap_or_else(|| panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key));
+ let mut partials = encoder
+ .allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32 * head_dim], DataType::F32))?;
+ let mut sums =
+ encoder.allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32], DataType::F32))?;
+ let mut maxs =
+ encoder.allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32], DataType::F32))?;
kernel_pass1.encode(
- queries_buf_borrow.deref(),
- key_cache_buffer,
- value_cache_buffer,
- partials_buf_borrow.deref_mut(),
- sums_buf_borrow.deref_mut(),
- maxs_buf_borrow.deref_mut(),
+ queries,
+ keys,
+ values,
+ &mut partials,
+ &mut sums,
+ &mut maxs,
gqa_factor as u32,
sequence_length as u32,
k_head_stride,
@@ -424,16 +426,16 @@ impl<B: Backend> Attention<B> {
scale,
num_heads as u32,
suffix_length as u32,
- trie_buffer,
+ trie_allocation,
self.sliding_window_size.map(|s| s as u32),
- sinks_buffer,
+ sinks_allocation,
encoder,
);
kernel_pass2.encode(
- partials_buf_borrow.deref(),
- sums_buf_borrow.deref(),
- maxs_buf_borrow.deref(),
- attention_output_buf_borrow.deref_mut(),
+ &partials,
+ &sums,
+ &maxs,
+ &mut attention_output,
num_heads as u32,
suffix_length as u32,
encoder,
@@ -441,23 +443,11 @@ impl<B: Backend> Attention<B> {
},
}
- if let Some(gate_kernel) = &self.gate_kernel {
- let gate_array = gate_array.as_ref().unwrap();
- let gate_buf_rc = gate_array.buffer();
- let gate_buf_borrow = gate_buf_rc.borrow();
- let total_elements = (suffix_length * num_heads * head_dim) as u32;
- gate_kernel.encode(
- gate_buf_borrow.deref(),
- attention_output_buf_borrow.deref_mut(),
- total_elements,
- encoder,
- );
- }
-
- Ok(())
+ Ok(attention_output)
}
}
+#[derive(Clone, Copy)]
enum KernelVariant {
Gemm,
SinglePass,
diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
index 37e9b1401..e260cf96b 100644
--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs
@@ -2,19 +2,16 @@ use std::rc::Rc;
use crate::{
DataType,
- backends::common::{Backend, Encoder},
+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder, Kernels, kernel::TensorAddSwapKernel},
config::TransformerLayerConfig,
encodable_block::{
- Attention, EncodingParameters, Linear, Mlp, Normalization, QKNorm, Rope, TensorAddSwap, TensorCopy,
+ Attention, AttentionArguments, EncodingParameters, LayerArguments, Linear, Mlp, Normalization, QKNorm, Rope,
},
- forward_pass::state::{ArrayId, ForwardPassState},
+ forward_pass::state::RopeType,
parameters::ParameterTree,
};
pub struct ClassifierLayer<B: Backend> {
- #[cfg_attr(not(feature = "tracing"), allow(dead_code))]
- layer_index: usize,
- copy_main_to_shortcut_mixer: TensorCopy<B>,
pre_attention_norm: Option<Normalization<B>>,
qkv_projection: Box<dyn Linear<B>>,
qk_norm: Option<QKNorm<B>>,
@@ -23,12 +20,15 @@ pub struct ClassifierLayer<B: Backend> {
attention: Attention<B>,
out_projection: Box<dyn Linear<B>>,
post_attention_norm: Option<Normalization<B>>,
- mixer_residual_add: TensorAddSwap<B>,
- copy_main_to_shortcut_mlp: TensorCopy<B>,
+ mixer_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,
pre_mlp_norm: Normalization<B>,
mlp: Box<dyn Mlp<B>>,
post_mlp_norm: Option<Normalization<B>>,
- mlp_residual_add: TensorAddSwap<B>,
+ mlp_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,
+ model_dim: usize,
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
}
impl<B: Backend> ClassifierLayer<B> {
@@ -47,12 +47,8 @@ impl<B: Backend> ClassifierLayer<B> {
) -> Self {
let ctx = context.as_ref(); // Reference for functions expecting &B::Context
let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
- let use_rope = attention_config.use_rope;
let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
- let copy_main_to_shortcut_mixer =
- TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();
-
let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {
if layer_loader.subtree("pre_mixer_norm").is_ok() {
Some(
@@ -60,8 +56,6 @@ impl<B: Backend> ClassifierLayer<B> {
ctx,
intermediate_data_type,
norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&layer_loader.subtree("pre_mixer_norm").unwrap(),
)
.expect("Failed to create pre-attention norm kernel"),
@@ -75,13 +69,10 @@ impl<B: Backend> ClassifierLayer<B> {
let qkv_projection = <dyn Linear<B>>::new(
&attention_config.qkv_projection_config,
- attention_config.has_qkv_biases,
model_dim,
[num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
ctx,
&layer_loader.subtree("mixer.qkv_projection").unwrap(),
- ArrayId::Main,
- ArrayId::QKV,
)
.expect("Failed to create qkv projection");
@@ -91,7 +82,6 @@ impl<B: Backend> ClassifierLayer<B> {
intermediate_data_type,
attention_config.query_norm_config.clone(),
attention_config.key_norm_config.clone(),
- ArrayId::QKV,
&layer_loader.subtree("mixer").unwrap(),
num_heads,
num_groups,
@@ -106,13 +96,10 @@ impl<B: Backend> ClassifierLayer<B> {
let out_projection = <dyn Linear<B>>::new(
&attention_config.out_projection_config,
- attention_config.has_out_biases,
num_heads * head_dim,
[model_dim],
ctx,
&layer_loader.subtree("mixer.out_projection").unwrap(),
- ArrayId::AttentionOutput,
- ArrayId::Main,
)
.expect("Failed to create out projection");
@@ -122,8 +109,6 @@ impl<B: Backend> ClassifierLayer<B> {
ctx,
intermediate_data_type,
norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&layer_loader.subtree("post_mixer_norm").unwrap(),
)
.expect("Failed to create post-attention norm kernel"),
@@ -132,18 +117,13 @@ impl<B: Backend> ClassifierLayer<B> {
None
};
- let mixer_residual_add =
- TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();
-
- let copy_main_to_shortcut_mlp =
- TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();
+ let mixer_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(ctx, intermediate_data_type)
+ .expect("Failed to create mixer residual add kernel");
let pre_mlp_norm = Normalization::new(
ctx,
intermediate_data_type,
layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&layer_loader.subtree("pre_mlp_norm").unwrap(),
)
.expect("Failed to create pre-MLP norm kernel");
@@ -165,8 +145,6 @@ impl<B: Backend> ClassifierLayer<B> {
ctx,
intermediate_data_type,
norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&layer_loader.subtree("post_mlp_norm").unwrap(),
)
.expect("Failed to create post-MLP norm kernel"),
@@ -178,7 +156,6 @@ impl<B: Backend> ClassifierLayer<B> {
let attention = Attention::new(
ctx,
intermediate_data_type,
- layer_index,
attention_scale,
attention_config.has_sinks,
false,
@@ -187,112 +164,163 @@ impl<B: Backend> ClassifierLayer<B> {
)
.expect("Failed to create attention kernel");
- let mlp_residual_add =
- TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();
+ let mlp_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(ctx, intermediate_data_type)
+ .expect("Failed to create mlp residual add kernel");
Self {
- layer_index,
- copy_main_to_shortcut_mixer,
pre_attention_norm,
qkv_projection,
qk_norm,
rope,
- use_rope,
+ use_rope: attention_config.use_rope,
attention,
out_projection,
post_attention_norm,
mixer_residual_add,
- copy_main_to_shortcut_mlp,
pre_mlp_norm,
mlp,
post_mlp_norm,
mlp_residual_add,
+ model_dim,
+ num_heads,
+ num_groups,
+ head_dim,
}
}
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ args: LayerArguments<B>,
parameters: &EncodingParameters,
+ main: Allocation<B>,
+ shortcut: &mut Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
+ ) -> Result<Allocation<B>, B::Error> {
+ let LayerArguments {
+ batch_dim,
+ token_positions,
+ token_subtrie_ranges,
+ attention_sinks,
+ rope_cosines,
+ rope_sines,
+ rope_max_sequence_length,
+ rope_dim,
+ #[cfg(feature = "tracing")]
+ trace,
+ ..
+ } = args;
#[cfg(feature = "tracing")]
- let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
+ let mut layer_traces = trace;
+ let layer_len = batch_dim * self.model_dim;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.inputs.allocation_mut(), ..);
}
- self.copy_main_to_shortcut_mixer.encode(state, encoder)?;
+ debug_assert_eq!(main.as_buffer_range_ref().range().len(), shortcut.as_buffer_range_ref().range().len());
+ encoder.encode_copy(&main, .., shortcut, ..);
- if let Some(ref pre_attn_norm) = self.pre_attention_norm {
- pre_attn_norm.encode(state, encoder)?;
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
- }
+ let mut main = if let Some(ref pre_attn_norm) = self.pre_attention_norm {
+ pre_attn_norm.encode(&main, 0, batch_dim, encoder)?
} else {
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
- }
+ main
+ };
+ #[cfg(feature = "tracing")]
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.pre_attention_norm.allocation_mut(), ..);
}
- self.qkv_projection.encode(state, encoder)?;
+ let mut qkv = self.qkv_projection.encode(main, batch_dim, encoder)?;
if let Some(ref qk_norm) = self.qk_norm {
- qk_norm.encode(state, encoder)?;
+ qk_norm.encode(&mut qkv, batch_dim, encoder)?;
}
- self.rope.encode(state, self.use_rope, encoder)?;
- self.attention.encode(state, parameters, encoder)?;
- self.out_projection.encode(state, encoder)?;
+ let cosines = rope_cosines.expect("Classifier attention layer requires RoPE cosine allocation");
+ let sines = rope_sines.expect("Classifier attention layer requires RoPE sine allocation");
+ let (queries, rotated_keys) = self.rope.encode(
+ &qkv,
+ token_positions,
+ cosines,
+ sines,
+ batch_dim,
+ self.num_heads,
+ self.num_groups,
+ self.head_dim,
+ rope_max_sequence_length,
+ rope_dim,
+ self.use_rope,
+ encoder,
+ )?;
+ let attention_output = self.attention.encode(
+ AttentionArguments {
+ projection_step: parameters.projection_step.unwrap_or(0),
+ token_subtrie_ranges,
+ attention_sinks,
+ kv_cache_layer: None,
+ },
+ &qkv,
+ &queries,
+ rotated_keys,
+ None,
+ batch_dim,
+ self.num_heads,
+ self.num_groups,
+ self.head_dim,
+ encoder,
+ )?;
+ main = self.out_projection.encode(attention_output, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.attention.allocation_mut(), ..);
}
if let Some(ref post_attn_norm) = self.post_attention_norm {
- post_attn_norm.encode(state, encoder)?;
+ main = post_attn_norm.encode(&main, 0, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.post_attention_norm.allocation_mut(), ..);
}
}
- self.mixer_residual_add.encode(state, encoder)?;
+ self.mixer_residual_add.encode(&mut *shortcut, &mut main, layer_len as u32, encoder);
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.mlp_inputs.allocation_mut(), ..);
}
- self.copy_main_to_shortcut_mlp.encode(state, encoder)?;
+ debug_assert_eq!(main.as_buffer_range_ref().range().len(), shortcut.as_buffer_range_ref().range().len());
+ encoder.encode_copy(&main, .., shortcut, ..);
- self.pre_mlp_norm.encode(state, encoder)?;
+ main = self.pre_mlp_norm.encode(&main, 0, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.pre_mlp_norm.allocation_mut(), ..);
}
- self.mlp.encode(state, encoder)?;
+ main = self.mlp.encode(main, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.mlp.allocation_mut(), ..);
}
if let Some(ref post_mlp_norm) = self.post_mlp_norm {
- post_mlp_norm.encode(state, encoder)?;
+ main = post_mlp_norm.encode(&main, 0, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.post_mlp_norm.allocation_mut(), ..);
}
}
- self.mlp_residual_add.encode(state, encoder)?;
+ self.mlp_residual_add.encode(shortcut, &mut main, layer_len as u32, encoder);
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&main, .., layer_traces.outputs.allocation_mut(), ..);
}
- Ok(())
+ Ok(main)
+ }
+
+ pub fn rope_type(&self) -> RopeType {
+ self.rope.rope_type()
}
}
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index a7565bc4c..e3d8f1948 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -4,14 +4,19 @@ use std::rc::Rc;
use thiserror::Error;
+#[cfg(feature = "tracing")]
+use crate::forward_pass::traces::ActivationTrace;
use crate::{
DataType,
- backends::common::{Backend, Encoder},
+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
config::{DecoderConfig, DecoderLayerType, MixerConfig},
- encodable_block::{Embedding, EncodingParameters, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError},
+ encodable_block::{
+ Embedding, EncodingParameters, LayerArguments, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError,
+ },
forward_pass::{
+ cache_layers::CacheLayers,
model_shape::ModelShape,
- state::{ArrayId, ForwardPassState, RopeType},
+ state::{RopeType, SharedBuffers},
},
parameters::ParameterTree,
};
@@ -31,6 +36,27 @@ pub struct Decoder<B: Backend> {
pub norm: RMSNorm<B>,
}
+pub struct DecoderArguments<'a, B: Backend> {
+ pub token_positions: &'a Allocation<B>,
+ pub token_parents: &'a Allocation<B>,
+ pub token_subtrie_ranges: Option<&'a Allocation<B>>,
+ pub shared_buffers: &'a SharedBuffers<B>,
+ pub cache_layers: Option<&'a mut CacheLayers<B>>,
+ pub batch_dim: usize,
+ pub sampling_start: usize,
+ pub sampling_length: usize,
+ pub rope_max_sequence_length: usize,
+ pub rope_dim: usize,
+ #[cfg(feature = "tracing")]
+ pub trace: Option<&'a mut ActivationTrace<B>>,
+}
+
+pub enum DecoderDecodeInput<'a, B: Backend> {
+ TokenIds(&'a Allocation<B>),
+ #[cfg(metal_backend)]
+ Embeddings(Allocation<B>),
+}
+
impl<B: Backend> Decoder<B> {
pub fn new(
context: &B::Context,
@@ -184,14 +210,11 @@ impl<B: Backend> Decoder<B> {
context,
norm_data_type,
decoder_config.output_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&decoder_weight_loader.subtree("output_norm").unwrap(),
None,
- Some(ArrayId::Shortcut),
+ true,
true,
)
- .map(RMSNorm::with_sampling_range)
.expect("Failed to create output RMS norm kernel");
(layers.into_boxed_slice(), norm_block)
@@ -218,35 +241,140 @@ impl<B: Backend> Decoder<B> {
})
}
- pub fn encode(
+ fn run_layers(
&self,
- state: &mut ForwardPassState<B>,
+ args: DecoderArguments<B>,
+ mut main: Allocation<B>,
parameters: &EncodingParameters,
encoder: &mut Encoder<B>,
- ) -> Result<(), DecoderError<B>> {
- self.embed.encode_lookup(state, encoder)?;
+ ) -> Result<(Allocation<B>, Allocation<B>), DecoderError<B>> {
+ let DecoderArguments {
+ token_positions,
+ token_parents,
+ token_subtrie_ranges,
+ shared_buffers,
+ mut cache_layers,
+ batch_dim,
+ sampling_start,
+ sampling_length,
+ rope_max_sequence_length,
+ rope_dim,
+ #[cfg(feature = "tracing")]
+ trace,
+ } = args;
+ #[cfg(feature = "tracing")]
+ let mut trace = trace;
+
+ let mut shortcut =
+ encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(DecoderError::BackendError)?;
for layer in self.layers.iter() {
- layer.encode(state, parameters, encoder).map_err(DecoderError::BackendError)?;
- }
+ let rope_type = layer.rope_type();
+ let rope_cosines = rope_type.and_then(|rope_type| shared_buffers.rope_cosines(rope_type));
+ let rope_sines = rope_type.and_then(|rope_type| shared_buffers.rope_sines(rope_type));
+ let attention_sinks = shared_buffers.attention_sinks(layer.layer_index);
+ #[cfg(feature = "tracing")]
+ let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);
- if state.is_prefilling() {
- return Ok(());
+ let cache_layer = cache_layers.as_deref_mut().map(|cache_layers| &mut cache_layers.data[layer.layer_index]);
+ main = layer
+ .encode(
+ LayerArguments {
+ batch_dim,
+ token_positions,
+ token_parents,
+ token_subtrie_ranges,
+ attention_sinks,
+ rope_cosines,
+ rope_sines,
+ rope_max_sequence_length,
+ rope_dim,
+ sampling_start,
+ sampling_length,
+ cache_layer,
+ #[cfg(feature = "tracing")]
+ trace: layer_trace,
+ },
+ parameters,
+ main,
+ &mut shortcut,
+ encoder,
+ )
+ .map_err(DecoderError::BackendError)?;
}
- self.norm.encode(state, encoder).map_err(DecoderError::BackendError)?;
+ Ok((main, shortcut))
+ }
+
+ pub fn encode_prefill(
+ &self,
+ args: DecoderArguments<B>,
+ token_ids: &Allocation<B>,
+ parameters: &EncodingParameters,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, DecoderError<B>> {
+ let main = self.embed.encode_lookup(token_ids, args.batch_dim, encoder)?;
+ let (main, _) = self.run_layers(args, main, parameters, encoder)?;
+ Ok(main)
+ }
+
+ pub fn encode_decode<'input>(
+ &self,
+ args: DecoderArguments<B>,
+ input: DecoderDecodeInput<'input, B>,
+ hidden_capture: Option<&mut Allocation<B>>,
+ parameters: &EncodingParameters,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, DecoderError<B>> {
+ let sampling_start = args.sampling_start;
+ let sampling_length = args.sampling_length;
+ let batch_dim = args.batch_dim;
+ #[cfg(feature = "tracing")]
+ let mut trace = args.trace;
+ let main = match input {
+ DecoderDecodeInput::TokenIds(token_ids) => self.embed.encode_lookup(token_ids, args.batch_dim, encoder)?,
+ #[cfg(metal_backend)]
+ DecoderDecodeInput::Embeddings(main) => main,
+ };
+ let (main, mut shortcut) = self.run_layers(
+ DecoderArguments {
+ token_positions: args.token_positions,
+ token_parents: args.token_parents,
+ token_subtrie_ranges: args.token_subtrie_ranges,
+ shared_buffers: args.shared_buffers,
+ cache_layers: args.cache_layers,
+ batch_dim: args.batch_dim,
+ sampling_start,
+ sampling_length,
+ rope_max_sequence_length: args.rope_max_sequence_length,
+ rope_dim: args.rope_dim,
+ #[cfg(feature = "tracing")]
+ trace: trace.as_deref_mut(),
+ },
+ main,
+ parameters,
+ encoder,
+ )?;
+
+ let output_norm = self
+ .norm
+ .encode(&main, sampling_start, sampling_length, Some(&mut shortcut), encoder)
+ .map_err(DecoderError::BackendError)?;
+ if let Some(hidden_capture) = hidden_capture {
+ let row_size_bytes = shortcut.as_buffer_range_ref().range().len() / batch_dim;
+ let input_offset = (batch_dim - 1) * row_size_bytes;
+ encoder.encode_copy(&shortcut, input_offset..input_offset + row_size_bytes, hidden_capture, ..);
+ }
#[cfg(feature = "tracing")]
- {
- let traces = state.traces().clone();
- state.encode_copy_array(encoder, ArrayId::Main, traces.borrow().output_norm.clone());
+ if let Some(trace) = trace.as_deref_mut() {
+ encoder.encode_copy(&output_norm, .., trace.output_norm.allocation_mut(), ..);
}
- self.embed.encode_readout(state, encoder)?;
+ let logits = self.embed.encode_readout(sampling_length, &output_norm, encoder)?;
#[cfg(feature = "tracing")]
- {
- let traces = state.traces().clone();
- state.encode_copy_array(encoder, ArrayId::Logits, traces.borrow().logits.clone());
+ if let Some(trace) = trace.as_deref_mut() {
+ encoder.encode_copy(&logits, .., trace.logits.allocation_mut(), ..);
}
- Ok(())
+ Ok(logits)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
index 2e13af21d..6e107ef21 100644
--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs
@@ -1,11 +1,10 @@
-use std::ops::{Deref, DerefMut};
-
use thiserror::Error;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{
Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,
DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
@@ -13,8 +12,8 @@ use crate::{
},
config::DeltaNetAttentionConfig,
encodable_block::linear::{Linear, LinearBlockError},
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::{ParameterLoaderError, ParameterTree, resolve_subtree},
+ forward_pass::delta_net_layer::DeltaNetLayer,
+ parameters::{ParameterLoaderError, ParameterTree, try_resolve_subtree},
};
#[derive(Debug, Error)]
@@ -30,7 +29,6 @@ pub enum DeltaNetMixerError<B: Backend> {
}
pub(crate) struct DeltaNetMixer<B: Backend> {
- layer_index: usize,
config: DeltaNetAttentionConfig,
in_projection: Box<dyn Linear<B>>,
out_projection: Box<dyn Linear<B>>,
@@ -44,21 +42,26 @@ pub(crate) struct DeltaNetMixer<B: Backend> {
delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,
norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,
// Parameters
- conv_weight: B::DenseBuffer,
- conv_bias: Option<B::DenseBuffer>,
- a_log: B::DenseBuffer,
- dt_bias: B::DenseBuffer,
- norm_weight: B::DenseBuffer,
+ conv_weight: Allocation<B>,
+ conv_bias: Option<Allocation<B>>,
+ a_log: Allocation<B>,
+ dt_bias: Allocation<B>,
+ norm_weight: Allocation<B>,
+ data_type: DataType,
+}
+
+pub(crate) struct DeltaNetArguments<'a, B: Backend> {
+ pub active_row_count: usize,
+ pub layer: &'a mut DeltaNetLayer<B>,
}
impl<B: Backend> DeltaNetMixer<B> {
pub(crate) fn new(
context: &B::Context,
config: DeltaNetAttentionConfig,
- layer_index: usize,
model_dim: usize,
decoder_layer_loader: &ParameterTree<B::Context>,
- ) -> Result<Self, DeltaNetMixerError<B>> {
+ ) -> Result<(Self, Option<Allocation<B>>), DeltaNetMixerError<B>> {
if config.kernel_size < 2 {
return Err(DeltaNetMixerError::UnsupportedConfiguration(format!(
"kernel_size must be >= 2, got {}",
@@ -82,44 +85,38 @@ impl<B: Backend> DeltaNetMixer<B> {
let has_bias = config.conv_config.has_biases;
// Load weights
- let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
- let conv_tree = resolve_subtree(&mixer_tree, &["conv", "conv1d"]);
+ let mixer_tree = try_resolve_subtree(decoder_layer_loader, &["mixer"])?;
+ let conv_tree = try_resolve_subtree(&mixer_tree, &["conv", "conv1d"])?;
- let in_projection = <dyn Linear<B>>::new(
+ let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&config.in_proj_config,
- false,
model_dim,
[config.total_proj_dim()],
context,
- &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),
- ArrayId::Main,
- ArrayId::SsmInProj,
+ &try_resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"])?,
)
.map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;
let out_projection = <dyn Linear<B>>::new(
&config.out_proj_config,
- false,
config.value_dim(),
[model_dim],
context,
- &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),
- ArrayId::AttentionOutput,
- ArrayId::Main,
+ &try_resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"])?,
)
.map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;
- let conv_weight = conv_tree.leaf("weights")?.read_buffer()?;
+ let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;
let conv_bias = if has_bias {
- Some(conv_tree.leaf("biases")?.read_buffer()?)
+ Some(conv_tree.leaf("biases")?.read_allocation()?)
} else {
None
};
- let a_log = mixer_tree.leaf("a_log")?.read_buffer()?;
- let dt_bias = mixer_tree.leaf("dt_bias")?.read_buffer()?;
- let norm_tree = resolve_subtree(&mixer_tree, &["norm", "inner_norm"]);
- let norm_weight = norm_tree.leaf("scales")?.read_buffer()?;
+ let a_log = mixer_tree.leaf("a_log")?.read_allocation()?;
+ let dt_bias = mixer_tree.leaf("dt_bias")?.read_allocation()?;
+ let norm_tree = try_resolve_subtree(&mixer_tree, &["norm", "inner_norm"])?;
+ let norm_weight = norm_tree.leaf("scales")?.read_allocation()?;
// Create kernels
let conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)
@@ -140,41 +137,41 @@ impl<B: Backend> DeltaNetMixer<B> {
let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, data_type)
.map_err(DeltaNetMixerError::BackendError)?;
- Ok(Self {
- layer_index,
- config,
- in_projection,
- out_projection,
- conv_update,
- delta_net_update,
- conv_pack,
- conv_scan,
- prefill_prep,
- delta_net_prefill,
- norm_gate,
- conv_weight,
- conv_bias,
- a_log,
- dt_bias,
- norm_weight,
- })
+ Ok((
+ Self {
+ config,
+ in_projection,
+ out_projection,
+ conv_update,
+ delta_net_update,
+ conv_pack,
+ conv_scan,
+ prefill_prep,
+ delta_net_prefill,
+ norm_gate,
+ conv_weight,
+ conv_bias,
+ a_log,
+ dt_bias,
+ norm_weight,
+ data_type,
+ },
+ in_projection_input_hadamard_factors,
+ ))
}
fn run_conv_update(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut DeltaNetLayer<B>,
+ in_proj: &mut Allocation<B>,
encoder: &mut Encoder<B>,
) {
- let in_proj = state.array(ArrayId::SsmInProj);
- let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));
-
let kernel_size = self.config.kernel_size;
-
self.conv_update.encode(
&self.conv_weight,
self.conv_bias.as_ref(),
- in_proj.buffer().borrow_mut().deref_mut(),
- conv_state.buffer().borrow_mut().deref_mut(),
+ in_proj,
+ &mut layer.conv_state,
kernel_size as u32,
self.config.conv_dim() as u32,
(kernel_size - 1) as u32,
@@ -184,24 +181,22 @@ impl<B: Backend> DeltaNetMixer<B> {
fn run_conv_scan(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut DeltaNetLayer<B>,
+ in_proj: &mut Allocation<B>,
encoder: &mut Encoder<B>,
suffix_length: usize,
- ) {
+ ) -> Result<(), B::Error> {
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size - 1;
let conv_dim = self.config.conv_dim();
let total_proj_dim = self.config.total_proj_dim();
-
- let in_proj = state.array(ArrayId::SsmInProj);
- let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));
-
- let padded = state.conv_padded_buffer().expect("Missing conv padded buffer");
+ let mut padded = encoder
+ .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.data_type))?;
self.conv_pack.encode(
- conv_state.buffer().borrow().deref(),
- in_proj.buffer().borrow().deref(),
- padded.buffer().borrow_mut().deref_mut(),
+ &layer.conv_state,
+ &*in_proj,
+ &mut padded,
state_stride as u32,
total_proj_dim as u32,
suffix_length as u32,
@@ -210,11 +205,11 @@ impl<B: Backend> DeltaNetMixer<B> {
);
self.conv_scan.encode(
- padded.buffer().borrow().deref(),
+ &padded,
&self.conv_weight,
self.conv_bias.as_ref(),
- in_proj.buffer().borrow_mut().deref_mut(),
- conv_state.buffer().borrow_mut().deref_mut(),
+ in_proj,
+ &mut layer.conv_state,
suffix_length as u32,
kernel_size as u32,
total_proj_dim as u32,
@@ -223,24 +218,25 @@ impl<B: Backend> DeltaNetMixer<B> {
total_proj_dim as u32,
encoder,
);
+ Ok(())
}
fn run_delta_rule(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut DeltaNetLayer<B>,
+ in_proj: &Allocation<B>,
+ active_row_count: usize,
encoder: &mut Encoder<B>,
- ) {
- let in_proj = state.array(ArrayId::SsmInProj);
- let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));
- let out = state.array(ArrayId::AttentionOutput);
-
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut out =
+ encoder.allocate_scratch(size_for_shape(&[active_row_count, self.config.value_dim()], self.data_type))?;
self.delta_net_update.encode(
- in_proj.buffer().borrow().deref(),
+ in_proj,
&self.a_log,
&self.dt_bias,
&self.norm_weight,
- ssm_state.buffer().borrow_mut().deref_mut(),
- out.buffer().borrow_mut().deref_mut(),
+ &mut layer.ssm_state,
+ &mut out,
self.config.num_heads as u32,
self.config.num_groups as u32,
self.config.value_head_dim as u32,
@@ -249,34 +245,36 @@ impl<B: Backend> DeltaNetMixer<B> {
self.config.norm_config.epsilon,
encoder,
);
+ Ok(out)
}
fn run_delta_rule_prefill(
&self,
- state: &mut ForwardPassState<B>,
- encoder: &mut Encoder<B>,
+ layer: &mut DeltaNetLayer<B>,
+ in_proj: &Allocation<B>,
suffix_length: usize,
- ) {
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
let num_dv_groups = ((self.config.value_head_dim + 7) / 8) as u32;
-
- let in_proj = state.array(ArrayId::SsmInProj);
- let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));
- let out = state.array(ArrayId::AttentionOutput);
-
- let aux = state.llm_aux.as_ref().expect("DeltaNet prep buffers not initialized");
- let prep_q_norm = aux.delta_net_prep_q_norm.as_ref().expect("DeltaNet prep_q_norm not initialized");
- let prep_k_norm = aux.delta_net_prep_k_norm.as_ref().expect("DeltaNet prep_k_norm not initialized");
- let prep_beta = aux.delta_net_prep_beta.as_ref().expect("DeltaNet prep_beta not initialized");
- let prep_decay = aux.delta_net_prep_decay.as_ref().expect("DeltaNet prep_decay not initialized");
+ let mut prep_q_norm =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;
+ let mut prep_k_norm =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;
+ let mut prep_beta =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;
+ let mut prep_decay =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;
+ let mut out =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;
self.prefill_prep.encode(
- in_proj.buffer().borrow().deref(),
+ in_proj,
&self.a_log,
&self.dt_bias,
- prep_q_norm.buffer().borrow_mut().deref_mut(),
- prep_k_norm.buffer().borrow_mut().deref_mut(),
- prep_beta.buffer().borrow_mut().deref_mut(),
- prep_decay.buffer().borrow_mut().deref_mut(),
+ &mut prep_q_norm,
+ &mut prep_k_norm,
+ &mut prep_beta,
+ &mut prep_decay,
self.config.num_heads as u32,
self.config.num_groups as u32,
self.config.key_dim() as u32,
@@ -286,13 +284,13 @@ impl<B: Backend> DeltaNetMixer<B> {
);
self.delta_net_prefill.encode(
- prep_q_norm.buffer().borrow().deref(),
- prep_k_norm.buffer().borrow().deref(),
- prep_beta.buffer().borrow().deref(),
- prep_decay.buffer().borrow().deref(),
- in_proj.buffer().borrow().deref(),
- ssm_state.buffer().borrow_mut().deref_mut(),
- out.buffer().borrow_mut().deref_mut(),
+ &prep_q_norm,
+ &prep_k_norm,
+ &prep_beta,
+ &prep_decay,
+ in_proj,
+ &mut layer.ssm_state,
+ &mut out,
self.config.num_heads as u32,
self.config.num_groups as u32,
self.config.value_head_dim as u32,
@@ -303,10 +301,9 @@ impl<B: Backend> DeltaNetMixer<B> {
encoder,
);
- // Norm gate
self.norm_gate.encode(
- out.buffer().borrow_mut().deref_mut(),
- in_proj.buffer().borrow().deref(),
+ &mut out,
+ in_proj,
&self.norm_weight,
self.config.num_heads as u32,
self.config.value_head_dim as u32,
@@ -317,29 +314,31 @@ impl<B: Backend> DeltaNetMixer<B> {
suffix_length as u32,
encoder,
);
+ Ok(out)
}
pub(crate) fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ args: DeltaNetArguments<B>,
+ input: Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let active_row_count = state.active_row_count();
- if active_row_count == 0 {
- return Ok(());
- }
-
- self.in_projection.encode(state, encoder)?;
-
- if active_row_count == 1 {
- self.run_conv_update(state, encoder);
- self.run_delta_rule(state, encoder);
+ ) -> Result<Allocation<B>, B::Error> {
+ let DeltaNetArguments {
+ active_row_count,
+ layer,
+ } = args;
+ assert!(active_row_count > 0, "DeltaNet mixer requires at least one active row");
+
+ let mut in_proj = self.in_projection.encode(input, active_row_count, encoder)?;
+
+ let delta_output = if active_row_count == 1 {
+ self.run_conv_update(layer, &mut in_proj, encoder);
+ self.run_delta_rule(layer, &in_proj, active_row_count, encoder)?
} else {
- self.run_conv_scan(state, encoder, active_row_count);
- self.run_delta_rule_prefill(state, encoder, active_row_count);
- }
+ self.run_conv_scan(layer, &mut in_proj, encoder, active_row_count)?;
+ self.run_delta_rule_prefill(layer, &in_proj, active_row_count, encoder)?
+ };
- self.out_projection.encode(state, encoder)?;
- Ok(())
+ self.out_projection.encode(delta_output, active_row_count, encoder)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 37d7de6f1..5d8e18bea 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -1,14 +1,11 @@
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
-};
+use std::cell::RefCell;
use thiserror::Error;
use crate::{
DataType,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{
FullPrecisionEmbeddingLookupKernel, ManualKernels, QuantizedEmbeddingLookupKernel,
matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
@@ -19,7 +16,6 @@ use crate::{
},
},
config::EmbeddingConfig,
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLeaf, ParameterLoaderError, ParameterTree},
};
@@ -46,14 +42,14 @@ pub enum EmbeddingError<B: Backend> {
enum TiedEmbeddingType<B: Backend> {
FullPrecision {
- weights: B::DenseBuffer,
+ weights: Allocation<B>,
lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
},
Quantized {
- weights: B::DenseBuffer,
- scales: B::DenseBuffer,
- biases: B::DenseBuffer,
+ weights: Allocation<B>,
+ scales: Allocation<B>,
+ biases: Allocation<B>,
lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
readout: QuantizedMatmulKernelEncodable<B>,
},
@@ -61,26 +57,26 @@ enum TiedEmbeddingType<B: Backend> {
enum UntiedEmbeddingLookupType<B: Backend> {
FullPrecision {
- weights: B::DenseBuffer,
+ weights: Allocation<B>,
lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
},
Quantized {
- weights: B::DenseBuffer,
- scales: B::DenseBuffer,
- biases: B::DenseBuffer,
+ weights: Allocation<B>,
+ scales: Allocation<B>,
+ biases: Allocation<B>,
lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
},
}
enum UntiedEmbeddingReadoutType<B: Backend> {
FullPrecision {
- weights: B::DenseBuffer,
+ weights: Allocation<B>,
readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
},
Quantized {
- weights: B::DenseBuffer,
- scales: B::DenseBuffer,
- biases: B::DenseBuffer,
+ weights: Allocation<B>,
+ scales: Allocation<B>,
+ biases: Allocation<B>,
readout: QuantizedMatmulKernelEncodable<B>,
},
}
@@ -98,6 +94,7 @@ enum EmbeddingTying<B: Backend> {
pub struct Embedding<B: Backend> {
tying: EmbeddingTying<B>,
input_scale: f32,
+ activation_data_type: DataType,
vocab_size: u32,
model_dim: u32,
}
@@ -151,6 +148,31 @@ fn validate_tensor<'file, 'context, 'leaf, B: Backend>(
}
impl<B: Backend> Embedding<B> {
+ fn activation_data_type(config: &EmbeddingConfig) -> DataType {
+ match config {
+ EmbeddingConfig::Tied {
+ precision,
+ ..
+ }
+ | EmbeddingConfig::Untied {
+ precision,
+ ..
+ } => (*precision).into(),
+ EmbeddingConfig::MLXQuantizedTied {
+ activation_precision,
+ ..
+ }
+ | EmbeddingConfig::MLXQuantizedUntied {
+ activation_precision,
+ ..
+ }
+ | EmbeddingConfig::MLXSemiQuantizedUntied {
+ activation_precision,
+ ..
+ } => (*activation_precision).into(),
+ }
+ }
+
pub fn new(
context: &B::Context,
vocab_size: u32,
@@ -209,7 +231,7 @@ impl<B: Backend> Embedding<B> {
let weights_leaf = lookup_tree.leaf("weights")?;
validate_tensor(&weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
- let weights = weights_leaf.read_buffer()?;
+ let weights = weights_leaf.read_allocation()?;
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
@@ -234,8 +256,8 @@ impl<B: Backend> Embedding<B> {
let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;
validate_tensor(&output_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
- let input_weights = input_weights_leaf.read_buffer()?;
- let output_weights = output_weights_leaf.read_buffer()?;
+ let input_weights = input_weights_leaf.read_allocation()?;
+ let output_weights = output_weights_leaf.read_allocation()?;
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
@@ -277,9 +299,9 @@ impl<B: Backend> Embedding<B> {
let biases_leaf = lookup_tree.leaf("biases")?;
validate_tensor(&biases_leaf, [vocab_size as usize, num_groups], data_type)?;
- let weights = weights_leaf.read_buffer()?;
- let scales = scales_leaf.read_buffer()?;
- let biases = biases_leaf.read_buffer()?;
+ let weights = weights_leaf.read_allocation()?;
+ let scales = scales_leaf.read_allocation()?;
+ let biases = biases_leaf.read_allocation()?;
let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
context,
@@ -353,13 +375,13 @@ impl<B: Backend> Embedding<B> {
let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;
validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
- let input_weights = input_weights_leaf.read_buffer()?;
- let input_scales = input_scales_leaf.read_buffer()?;
- let input_biases = input_biases_leaf.read_buffer()?;
+ let input_weights = input_weights_leaf.read_allocation()?;
+ let input_scales = input_scales_leaf.read_allocation()?;
+ let input_biases = input_biases_leaf.read_allocation()?;
- let output_weights = output_weights_leaf.read_buffer()?;
- let output_scales = output_scales_leaf.read_buffer()?;
- let output_biases = output_biases_leaf.read_buffer()?;
+ let output_weights = output_weights_leaf.read_allocation()?;
+ let output_scales = output_scales_leaf.read_allocation()?;
+ let output_biases = output_biases_leaf.read_allocation()?;
let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
context,
@@ -430,11 +452,11 @@ impl<B: Backend> Embedding<B> {
let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;
validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
- let input_weights = input_weights_leaf.read_buffer()?;
+ let input_weights = input_weights_leaf.read_allocation()?;
- let output_weights = output_weights_leaf.read_buffer()?;
- let output_scales = output_scales_leaf.read_buffer()?;
- let output_biases = output_biases_leaf.read_buffer()?;
+ let output_weights = output_weights_leaf.read_allocation()?;
+ let output_scales = output_scales_leaf.read_allocation()?;
+ let output_biases = output_biases_leaf.read_allocation()?;
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
.map_err(EmbeddingError::BackendError)?;
@@ -473,6 +495,7 @@ impl<B: Backend> Embedding<B> {
};
let input_scale = common.input_scale.unwrap_or(1.0);
+ let activation_data_type = Self::activation_data_type(config);
if let Some(logit_soft_cap) = common.logit_soft_cap {
return Err(EmbeddingError::UnsupportedConfiguration(format!("logit_soft_cap={logit_soft_cap:?}")));
@@ -481,6 +504,7 @@ impl<B: Backend> Embedding<B> {
Ok(Self {
tying,
input_scale,
+ activation_data_type,
vocab_size,
model_dim,
})
@@ -488,20 +512,17 @@ impl<B: Backend> Embedding<B> {
pub fn encode_lookup(
&self,
- state: &mut ForwardPassState<B>,
+ token_ids: &Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), EmbeddingError<B>> {
- let batch_dim = state.active_row_count() as u32;
-
- let token_ids_array = state.array(ArrayId::TokenIds);
- let token_ids_buffer_rc = token_ids_array.buffer();
- let token_ids_buffer_borrow = token_ids_buffer_rc.borrow();
- let token_ids = token_ids_buffer_borrow.deref();
-
- let output_array = state.array(ArrayId::Main);
- let output_buffer_rc = output_array.buffer();
- let mut output_buffer_borrow = output_buffer_rc.borrow_mut();
- let output = output_buffer_borrow.deref_mut();
+ ) -> Result<Allocation<B>, EmbeddingError<B>> {
+ let mut output = encoder
+ .allocate_scratch(crate::array::size_for_shape(
+ &[batch_dim, self.model_dim as usize],
+ self.activation_data_type,
+ ))
+ .map_err(EmbeddingError::BackendError)?;
+ let batch_dim = batch_dim as u32;
match &self.tying {
EmbeddingTying::Tied {
@@ -522,7 +543,7 @@ impl<B: Backend> Embedding<B> {
} => lookup.encode(
token_ids,
weights,
- output,
+ &mut output,
batch_dim,
self.vocab_size,
self.model_dim,
@@ -554,7 +575,7 @@ impl<B: Backend> Embedding<B> {
weights,
scales,
biases,
- output,
+ &mut output,
batch_dim,
self.vocab_size,
self.model_dim,
@@ -564,31 +585,22 @@ impl<B: Backend> Embedding<B> {
},
};
- Ok(())
+ Ok(output)
}
pub fn encode_readout(
&self,
- state: &mut ForwardPassState<B>,
+ batch_dim: usize,
+ input_allocation: &Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), EmbeddingError<B>> {
- let batch_dim = state.sampling_length();
-
- if batch_dim == 0 {
- return Ok(());
- }
-
- let input_array = state.array(ArrayId::Main);
- let input_buffer_rc = input_array.buffer();
- let input_buffer_borrow = input_buffer_rc.borrow();
- let input = input_buffer_borrow.deref();
-
- let output_array = state.array(ArrayId::Logits);
- let output_buffer_rc = output_array.buffer();
- let mut output_buffer_borrow = output_buffer_rc.borrow_mut();
- let output = output_buffer_borrow.deref_mut();
-
- let input_offset = state.sampling_start() * self.model_dim as usize * input_array.data_type().size_in_bytes();
+ ) -> Result<Allocation<B>, EmbeddingError<B>> {
+ assert!(batch_dim > 0, "Embedding readout requires at least one row");
+ let mut output_allocation = encoder
+ .allocate_scratch(crate::array::size_for_shape(
+ &[batch_dim, self.vocab_size as usize],
+ self.activation_data_type,
+ ))
+ .map_err(EmbeddingError::BackendError)?;
match &self.tying {
EmbeddingTying::Tied {
@@ -610,14 +622,13 @@ impl<B: Backend> Embedding<B> {
let input_dim = self.model_dim as usize;
let output_dim = self.vocab_size as usize;
readout.borrow_mut().encode(
- state.context(),
MatmulArguments {
- a: input,
- a_offset: input_offset as u64,
+ a: input_allocation,
+ a_offset: 0,
b: weights,
ab_scale: 1.0,
c: MatmulArgumentC::None,
- d: output,
+ d: &mut output_allocation,
batch_dim: batch_dim as u32,
input_dim: input_dim as u32,
output_dim: output_dim as u32,
@@ -648,19 +659,19 @@ impl<B: Backend> Embedding<B> {
readout.encode(
encoder,
QuantizedMatmulArguments {
- a_buffer: input,
- a_offset: input_offset,
- b_buffer: weights,
- scales_buffer: scales,
- zero_points_or_biases_buffer: biases,
- output_buffer: output,
+ a: input_allocation,
+ a_offset: 0,
+ b: weights,
+ scales,
+ zero_points_or_biases: biases,
+ output: &mut output_allocation,
hadamard_factors: None,
batch_dim,
},
- )?;
+ );
},
};
- Ok(())
+ Ok(output_allocation)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rs
index b8dfe25a6..0bb930cb5 100644
--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs
@@ -1,26 +1,26 @@
//! Layer executables - a single decoder layer with mixer, norms, and MLP.
-#[cfg(feature = "tracing")]
-use std::ops::{Deref, DerefMut};
use std::rc::Rc;
use super::MixerExecutables;
#[cfg(feature = "tracing")]
use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};
+#[cfg(feature = "tracing")]
+use crate::forward_pass::traces::LayerActivationTrace;
use crate::{
DataType,
- backends::common::{Backend, Encoder},
+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},
config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},
encodable_block::{
- Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,
+ Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, EncodingParameters, Linear, MambaArguments,
+ MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,
},
- forward_pass::state::{ArrayId, ForwardPassState},
+ forward_pass::{cache_layers::CacheLayer, state::RopeType},
parameters::ParameterTree,
};
/// A single decoder layer with all its components.
pub struct LayerExecutables<B: Backend> {
- #[cfg(feature = "tracing")]
pub layer_index: usize,
#[cfg(feature = "tracing")]
pub tensor_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
@@ -30,6 +30,8 @@ pub struct LayerExecutables<B: Backend> {
pub pre_mlp_norm: RMSNorm<B>,
pub mlp: Box<dyn Mlp<B>>,
pub post_mlp_norm: Option<RMSNorm<B>>,
+ #[cfg(feature = "tracing")]
+ model_dim: usize,
}
impl<B: Backend> LayerExecutables<B> {
@@ -62,7 +64,6 @@ impl<B: Backend> LayerExecutables<B> {
let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {
MixerConfig::Attention(attention_config) => {
let rope_block = rope.expect("RoPE encoder missing for attention layer");
- let use_rope = attention_config.use_rope;
let layer_num_heads = attention_config.num_heads.unwrap_or(num_heads);
let layer_num_groups = attention_config.num_groups.unwrap_or(num_groups);
@@ -73,38 +74,58 @@ impl<B: Backend> LayerExecutables<B> {
let (qkv_projection, input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&attention_config.qkv_projection_config,
- attention_config.has_qkv_biases,
model_dim,
[q_dim, kv_dim, kv_dim],
context,
&decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),
- ArrayId::Main,
- ArrayId::QKV,
)
.expect("Failed to create qkv projection");
let has_gate = attention_config.has_gate || attention_config.gate_projection_config.is_some();
- let gate_projection = if has_gate {
+ let gate_projection = has_gate.then(|| {
let gate_config = attention_config
.gate_projection_config
.as_ref()
.unwrap_or(&attention_config.qkv_projection_config);
- Some(
- <dyn Linear<B>>::new(
- gate_config,
+ let gate_tree = decoder_layer_loader.subtree("mixer.gate_projection").unwrap();
+ match (input_hadamard_factors.is_some(), gate_config) {
+ (
+ true,
+ crate::config::LinearConfig::RHTLinearWrapper {
+ inner_config,
+ ..
+ },
+ ) => {
+ let output_factors = gate_tree
+ .leaf("output_factors")
+ .expect("Failed to get gate projection output_factors")
+ .read_allocation()
+ .expect("Failed to read gate projection output_factors");
+ let inner_tree = gate_tree
+ .subtree("inner_linear")
+ .expect("Failed to get gate projection inner_linear subtree");
+ <dyn Linear<B>>::new_with_output_hadamard(
+ context,
+ inner_config,
+ &inner_tree,
+ output_factors,
+ model_dim,
+ q_dim,
+ )
+ },
+ (
false,
- model_dim,
- [q_dim],
- context,
- &decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),
- ArrayId::Main,
- ArrayId::Gate,
+ crate::config::LinearConfig::RHTLinearWrapper {
+ ..
+ },
)
- .expect("Failed to create gate projection"),
- )
- } else {
- None
- };
+ | (true, _) => {
+ panic!("attention qkv/gate projections must share input hadamard")
+ },
+ (false, _) => <dyn Linear<B>>::new(gate_config, model_dim, [q_dim], context, &gate_tree),
+ }
+ .expect("Failed to create gate projection")
+ });
let qk_norm =
if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
@@ -113,7 +134,6 @@ impl<B: Backend> LayerExecutables<B> {
intermediate_data_type,
attention_config.query_norm_config.clone(),
attention_config.key_norm_config.clone(),
- ArrayId::QKV,
&decoder_layer_loader.subtree("mixer").unwrap(),
layer_num_heads,
layer_num_groups,
@@ -128,20 +148,16 @@ impl<B: Backend> LayerExecutables<B> {
let out_projection = <dyn Linear<B>>::new(
&attention_config.out_projection_config,
- attention_config.has_out_biases,
q_dim,
[model_dim],
context,
&decoder_layer_loader.subtree("mixer.out_projection").unwrap(),
- ArrayId::AttentionOutput,
- ArrayId::Main,
)
.expect("Failed to create out projection");
let attention = Attention::new(
context,
intermediate_data_type,
- layer_index,
attention_scale,
attention_config.has_sinks,
attention_config.is_causal.unwrap_or(true),
@@ -156,30 +172,31 @@ impl<B: Backend> LayerExecutables<B> {
gate_projection,
qk_norm,
rope: rope_block,
- use_rope,
+ use_rope: attention_config.use_rope,
attention,
out_projection,
+ num_heads: layer_num_heads,
+ num_groups: layer_num_groups,
+ head_dim: layer_head_dim,
},
input_hadamard_factors,
)
},
MixerConfig::Mamba(mamba_config) => {
- let mixer = MambaMixer::new(
+ let (mixer, input_hadamard_factors) = MambaMixer::new(
context,
layer_type.clone(),
mamba_config.clone(),
layer_index,
model_dim,
- num_heads,
- head_dim,
- num_groups,
decoder_layer_loader,
- );
+ )
+ .expect("Failed to create Mamba mixer");
(
MixerExecutables::StateSpace {
mixer,
},
- None,
+ input_hadamard_factors,
)
},
MixerConfig::ShortConv(short_conv_config) => {
@@ -190,7 +207,8 @@ impl<B: Backend> LayerExecutables<B> {
layer_index,
model_dim,
decoder_layer_loader,
- );
+ )
+ .expect("Failed to create ShortConv mixer");
(
MixerExecutables::ShortConv {
mixer,
@@ -199,14 +217,14 @@ impl<B: Backend> LayerExecutables<B> {
)
},
MixerConfig::DeltaNet(delta_net_config) => {
- let mixer =
- DeltaNetMixer::new(context, delta_net_config.clone(), layer_index, model_dim, decoder_layer_loader)
+ let (mixer, input_hadamard_factors) =
+ DeltaNetMixer::new(context, delta_net_config.clone(), model_dim, decoder_layer_loader)
.expect("Failed to create DeltaNet mixer");
(
MixerExecutables::DeltaNet {
mixer,
},
- None,
+ input_hadamard_factors,
)
},
};
@@ -215,11 +233,9 @@ impl<B: Backend> LayerExecutables<B> {
context,
intermediate_data_type,
layer_config.pre_attention_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
mixer_hadamard_factors,
- Some(ArrayId::Shortcut),
+ true,
layer_index > 0,
)
.expect("Failed to create RMS norm kernel");
@@ -230,11 +246,9 @@ impl<B: Backend> LayerExecutables<B> {
context,
intermediate_data_type,
norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
None,
- None,
+ false,
false,
)
.expect("Failed to create RMS norm kernel"),
@@ -256,11 +270,9 @@ impl<B: Backend> LayerExecutables<B> {
context,
intermediate_data_type,
layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
mlp_input_hadamard_factors,
- Some(ArrayId::Shortcut),
+ true,
true,
)
.expect("Failed to create RMS norm kernel");
@@ -271,11 +283,9 @@ impl<B: Backend> LayerExecutables<B> {
context,
intermediate_data_type,
norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
None,
- None,
+ false,
false,
)
.expect("Failed to create RMS norm kernel"),
@@ -285,7 +295,6 @@ impl<B: Backend> LayerExecutables<B> {
};
Self {
- #[cfg(feature = "tracing")]
layer_index,
#[cfg(feature = "tracing")]
tensor_add,
@@ -295,26 +304,46 @@ impl<B: Backend> LayerExecutables<B> {
pre_mlp_norm,
mlp,
post_mlp_norm,
+ #[cfg(feature = "tracing")]
+ model_dim,
}
}
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ args: LayerArguments<B>,
parameters: &EncodingParameters,
+ input: Allocation<B>,
+ shortcut: &mut Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
+ ) -> Result<Allocation<B>, B::Error> {
+ let LayerArguments {
+ batch_dim,
+ token_positions,
+ token_parents,
+ token_subtrie_ranges,
+ attention_sinks,
+ rope_cosines,
+ rope_sines,
+ rope_max_sequence_length,
+ rope_dim,
+ sampling_start,
+ sampling_length,
+ mut cache_layer,
+ #[cfg(feature = "tracing")]
+ trace,
+ } = args;
#[cfg(feature = "tracing")]
- let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
+ let mut layer_traces = trace;
- self.pre_attention_norm.encode(state, encoder)?;
+ let mut hidden = self.pre_attention_norm.encode(&input, 0, batch_dim, Some(shortcut), encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().inputs.clone());
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(shortcut, .., layer_traces.inputs.allocation_mut(), ..);
+ encoder.encode_copy(&hidden, .., layer_traces.pre_attention_norm.allocation_mut(), ..);
}
- match &self.mixer {
+ hidden = match &self.mixer {
MixerExecutables::Attention {
qkv_projection,
gate_projection,
@@ -323,100 +352,201 @@ impl<B: Backend> LayerExecutables<B> {
use_rope,
attention,
out_projection,
+ num_heads,
+ num_groups,
+ head_dim,
} => {
- qkv_projection.encode(state, encoder)?;
- if let Some(gate_proj) = gate_projection {
- gate_proj.encode(state, encoder)?;
- }
+ let gate_input = if gate_projection.is_some() {
+ let hidden_len = hidden.as_buffer_range_ref().range().len();
+ let mut gate_input = encoder.allocate_scratch(hidden_len)?;
+ encoder.encode_copy(&hidden, .., &mut gate_input, ..);
+ Some(gate_input)
+ } else {
+ None
+ };
+ let mut qkv = qkv_projection.encode(hidden, batch_dim, encoder)?;
+ let gate = match (gate_projection, gate_input) {
+ (Some(gate_proj), Some(gate_input)) => Some(gate_proj.encode(gate_input, batch_dim, encoder)?),
+ _ => None,
+ };
if let Some(norm) = qk_norm {
- norm.encode(state, encoder)?;
- }
- rope.encode(state, *use_rope, encoder)?;
- attention.encode(state, parameters, encoder)?;
- out_projection.encode(state, encoder)?;
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
+ norm.encode(&mut qkv, batch_dim, encoder)?;
}
+ let cosines = rope_cosines.expect("Attention layer requires RoPE cosine allocation");
+ let sines = rope_sines.expect("Attention layer requires RoPE sine allocation");
+ let (queries, rotated_keys) = rope.encode(
+ &qkv,
+ token_positions,
+ cosines,
+ sines,
+ batch_dim,
+ *num_heads,
+ *num_groups,
+ *head_dim,
+ rope_max_sequence_length,
+ rope_dim,
+ *use_rope,
+ encoder,
+ )?;
+ let kv_cache_layer = cache_layer
+ .as_deref_mut()
+ .map(|layer| layer.as_transformer_mut().expect("Attention layer expects transformer cache"));
+ let attention_output = attention.encode(
+ AttentionArguments {
+ projection_step: parameters.projection_step.unwrap_or(0),
+ token_subtrie_ranges,
+ attention_sinks,
+ kv_cache_layer,
+ },
+ &qkv,
+ &queries,
+ rotated_keys,
+ gate.as_ref(),
+ batch_dim,
+ *num_heads,
+ *num_groups,
+ *head_dim,
+ encoder,
+ )?;
+ out_projection.encode(attention_output, batch_dim, encoder)?
},
MixerExecutables::StateSpace {
mixer,
} => {
- mixer.encode(state, encoder)?;
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
- }
+ let layer = cache_layer
+ .as_deref_mut()
+ .expect("State-space layer requires cache state")
+ .as_state_space_mut()
+ .expect("State-space mixer expects SSM cache layer");
+ mixer.encode(
+ MambaArguments {
+ active_row_count: batch_dim,
+ layer,
+ },
+ hidden,
+ encoder,
+ )?
},
MixerExecutables::ShortConv {
mixer,
} => {
- mixer.encode(state, encoder)?;
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
- }
+ let layer = cache_layer
+ .as_deref_mut()
+ .expect("ShortConv layer requires cache state")
+ .as_short_conv_mut()
+ .expect("ShortConv mixer expects ShortConv cache layer");
+ mixer.encode(
+ ShortConvArguments {
+ active_row_count: batch_dim,
+ sampling_start,
+ sampling_length,
+ token_parents,
+ layer,
+ },
+ hidden,
+ encoder,
+ )?
},
MixerExecutables::DeltaNet {
mixer,
} => {
- mixer.encode(state, encoder)?;
- #[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
- }
+ let layer = cache_layer
+ .as_deref_mut()
+ .expect("DeltaNet layer requires cache state")
+ .as_delta_net_mut()
+ .expect("DeltaNet mixer expects DeltaNet cache layer");
+ mixer.encode(
+ DeltaNetArguments {
+ active_row_count: batch_dim,
+ layer,
+ },
+ hidden,
+ encoder,
+ )?
},
+ };
+ #[cfg(feature = "tracing")]
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&hidden, .., layer_traces.attention.allocation_mut(), ..);
}
if let Some(post_attention_norm) = &self.post_attention_norm {
- post_attention_norm.encode(state, encoder)?;
+ hidden = post_attention_norm.encode(&hidden, 0, batch_dim, None, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&hidden, .., layer_traces.post_attention_norm.allocation_mut(), ..);
}
}
- // main = attention_result
- self.pre_mlp_norm.encode(state, encoder)?;
- // shortcut = attention_result + shortcut; main = normalized(shortcut)
+ hidden = self.pre_mlp_norm.encode(&hidden, 0, batch_dim, Some(shortcut), encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().mlp_inputs.clone());
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(shortcut, .., layer_traces.mlp_inputs.allocation_mut(), ..);
+ encoder.encode_copy(&hidden, .., layer_traces.pre_mlp_norm.allocation_mut(), ..);
}
- self.mlp.encode(state, encoder)?;
+ hidden = self.mlp.encode(hidden, batch_dim, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&hidden, .., layer_traces.mlp.allocation_mut(), ..);
}
if let Some(post_mlp_norm) = &self.post_mlp_norm {
- post_mlp_norm.encode(state, encoder)?;
+ hidden = post_mlp_norm.encode(&hidden, 0, batch_dim, None, encoder)?;
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ encoder.encode_copy(&hidden, .., layer_traces.post_mlp_norm.allocation_mut(), ..);
}
}
- // main = mlp_result
- // next layer's pre_attention_norm (or output_norm) fuses the residual add
#[cfg(feature = "tracing")]
- if let Some(ref layer_traces) = layer_traces {
- let size = state.array(ArrayId::Main).shape().into_iter().copied().product::<usize>() as u32;
- let input_a = state.array(ArrayId::Main).buffer();
- let input_b = state.array(ArrayId::Shortcut).buffer();
- let output = layer_traces.borrow().outputs.buffer();
+ if let Some(layer_traces) = layer_traces.as_deref_mut() {
+ let size = (batch_dim * self.model_dim) as u32;
self.tensor_add.encode(
- Some(input_a.borrow().deref()),
- input_b.borrow().deref(),
- output.borrow_mut().deref_mut(),
+ Some(&hidden),
+ &*shortcut,
+ layer_traces.outputs.allocation_mut(),
size,
size,
encoder,
);
}
- Ok(())
+ Ok(hidden)
}
+
+ pub fn rope_type(&self) -> Option<RopeType> {
+ match &self.mixer {
+ MixerExecutables::Attention {
+ rope,
+ ..
+ } => Some(rope.rope_type()),
+ MixerExecutables::StateSpace {
+ ..
+ }
+ | MixerExecutables::ShortConv {
+ ..
+ }
+ | MixerExecutables::DeltaNet {
+ ..
+ } => None,
+ }
+ }
+}
+
+pub struct LayerArguments<'a, B: Backend> {
+ pub batch_dim: usize,
+ pub token_positions: &'a Allocation<B>,
+ pub token_parents: &'a Allocation<B>,
+ pub token_subtrie_ranges: Option<&'a Allocation<B>>,
+ pub attention_sinks: Option<&'a Allocation<B>>,
+ pub rope_cosines: Option<&'a Allocation<B>>,
+ pub rope_sines: Option<&'a Allocation<B>>,
+ pub rope_max_sequence_length: usize,
+ pub rope_dim: usize,
+ pub sampling_start: usize,
+ pub sampling_length: usize,
+ pub cache_layer: Option<&'a mut CacheLayer<B>>,
+ #[cfg(feature = "tracing")]
+ pub trace: Option<&'a mut LayerActivationTrace<B>>,
}
diff --git a/crates/backend-uzu/src/encodable_block/layer/mixer.rs b/crates/backend-uzu/src/encodable_block/layer/mixer.rs
index e411bd700..c5b0db2d9 100644
--- a/crates/backend-uzu/src/encodable_block/layer/mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/mixer.rs
@@ -17,6 +17,9 @@ pub(crate) enum MixerExecutables<B: Backend> {
use_rope: bool,
attention: Attention<B>,
out_projection: Box<dyn Linear<B>>,
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
},
StateSpace {
mixer: MambaMixer<B>,
diff --git a/crates/backend-uzu/src/encodable_block/layer/mod.rs b/crates/backend-uzu/src/encodable_block/layer/mod.rs
index 5aea3c675..70b5019cb 100644
--- a/crates/backend-uzu/src/encodable_block/layer/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/layer/mod.rs
@@ -3,5 +3,5 @@
mod executables;
mod mixer;
-pub use executables::LayerExecutables;
+pub use executables::{LayerArguments, LayerExecutables};
pub(crate) use mixer::MixerExecutables;
diff --git a/crates/backend-uzu/src/encodable_block/layer_norm.rs b/crates/backend-uzu/src/encodable_block/layer_norm.rs
index 3469a7d1f..90a08cd16 100644
--- a/crates/backend-uzu/src/encodable_block/layer_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/layer_norm.rs
@@ -1,21 +1,15 @@
//! LayerNorm encodable.
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
use thiserror::Error;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{Kernels, LayerNormKernel},
},
config::{NormalizationConfig, UpcastMode},
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -30,9 +24,10 @@ pub enum LayerNormError<B: Backend> {
pub struct LayerNorm<B: Backend> {
kernel: <B::Kernels as Kernels>::LayerNormKernel,
config: NormalizationConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- scales_buffer: Rc<RefCell<B::DenseBuffer>>,
+ scales: Allocation<B>,
+ element_count: usize,
+ input_data_type: DataType,
+ output_data_type: DataType,
}
impl<B: Backend> LayerNorm<B> {
@@ -40,64 +35,61 @@ impl<B: Backend> LayerNorm<B> {
context: &B::Context,
intermediate_data_type: DataType,
config: NormalizationConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
) -> Result<Self, LayerNormError<B>> {
- let scales = parameter_tree.leaf_array("scales").map_err(LayerNormError::ParameterError)?;
+ let scales_leaf = parameter_tree.leaf("scales").map_err(LayerNormError::ParameterError)?;
+ let element_count = scales_leaf.shape()[0];
+ let scales = scales_leaf.read_allocation().map_err(LayerNormError::ParameterError)?;
let accumulation_data_type: DataType = config.accumulation_precision.into();
let scale_data_type: DataType = config.scale_precision.into();
-
let kernel = <B::Kernels as Kernels>::LayerNormKernel::new(
context,
intermediate_data_type,
scale_data_type,
scale_data_type,
accumulation_data_type,
- input_array_id == output_array_id,
+ false,
)
.map_err(LayerNormError::BackendError)?;
Ok(Self {
kernel,
config,
- input_array_id,
- output_array_id,
- scales_buffer: scales.buffer(),
+ scales,
+ element_count,
+ input_data_type: intermediate_data_type,
+ output_data_type: scale_data_type,
})
}
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: &Allocation<B>,
+ row_offset: usize,
+ row_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let input_array = state.array(self.input_array_id);
- let output_array = state.array(self.output_array_id);
-
- let batch_dim = input_array.shape()[0] as u32;
- let model_dim = input_array.shape()[1] as u32;
+ ) -> Result<Allocation<B>, B::Error> {
let full_layer = if self.config.upcast_mode == UpcastMode::FullLayer {
1u32
} else {
0u32
};
-
- let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());
- let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());
-
+ let row_size = self.element_count * self.input_data_type.size_in_bytes();
+ let input_offset = row_offset * row_size;
+ let mut output =
+ encoder.allocate_scratch(size_for_shape(&[row_count, self.element_count], self.output_data_type))?;
self.kernel.encode(
- input_buffer_borrow.as_deref(),
- self.scales_buffer.borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- batch_dim,
- model_dim,
+ Some((input, input_offset)),
+ &self.scales,
+ &mut output,
+ row_count as u32,
+ self.element_count as u32,
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
full_layer,
encoder,
);
- Ok(())
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs b/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
index 065c43d44..32c2ec921 100644
--- a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/full_precision.rs
@@ -1,22 +1,18 @@
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
+use std::cell::RefCell;
use thiserror::Error;
use super::Linear;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{
ManualKernels,
matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
},
},
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -53,12 +49,11 @@ pub enum FullPrecisionLinearError<B: Backend> {
pub struct FullPrecisionLinear<B: Backend> {
kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
- bias_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,
- weights_buffer: Rc<RefCell<B::DenseBuffer>>,
+ bias: Option<Allocation<B>>,
+ weights: Allocation<B>,
input_dim: usize,
output_dim: usize,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
+ precision: DataType,
}
impl<B: Backend> FullPrecisionLinear<B> {
@@ -68,15 +63,13 @@ impl<B: Backend> FullPrecisionLinear<B> {
input_dim: usize,
output_dim: usize,
parameter_tree: &ParameterTree<B::Context>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
) -> Result<Self, FullPrecisionLinearError<B>> {
if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(FullPrecisionLinearError::UnsupportedDataType(precision));
}
- let weights = parameter_tree.leaf_array("weights").map_err(FullPrecisionLinearError::ParameterError)?;
- let weights_shape = weights.shape().to_vec();
+ let weights_leaf = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;
+ let weights_shape = weights_leaf.shape().to_vec();
if weights_shape != [output_dim, input_dim] {
return Err(FullPrecisionLinearError::InvalidWeightsShape {
got: weights_shape.into_boxed_slice(),
@@ -85,16 +78,16 @@ impl<B: Backend> FullPrecisionLinear<B> {
});
}
- if weights.data_type() != precision {
+ if weights_leaf.data_type() != precision {
return Err(FullPrecisionLinearError::InvalidWeightsDataType {
expected: precision,
- got: weights.data_type(),
+ got: weights_leaf.data_type(),
});
}
- let bias_buffer = match parameter_tree.leaf_array("biases") {
- Ok(biases) => {
- let bias_shape = biases.shape().to_vec();
+ let bias = match parameter_tree.leaf("biases") {
+ Ok(biases_leaf) => {
+ let bias_shape = biases_leaf.shape().to_vec();
if bias_shape != [output_dim] {
return Err(FullPrecisionLinearError::InvalidBiasShape {
got: bias_shape.into_boxed_slice(),
@@ -102,14 +95,14 @@ impl<B: Backend> FullPrecisionLinear<B> {
});
}
- if biases.data_type() != precision {
+ if biases_leaf.data_type() != precision {
return Err(FullPrecisionLinearError::InvalidBiasDataType {
expected: precision,
- got: biases.data_type(),
+ got: biases_leaf.data_type(),
});
}
- Some(biases.buffer())
+ Some(biases_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?)
},
Err(_) => None,
};
@@ -118,12 +111,11 @@ impl<B: Backend> FullPrecisionLinear<B> {
Ok(Self {
kernel: RefCell::new(kernel),
- bias_buffer,
- weights_buffer: weights.buffer(),
+ bias,
+ weights: weights_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?,
input_dim,
output_dim,
- input_array_id,
- output_array_id,
+ precision,
})
}
}
@@ -131,32 +123,28 @@ impl<B: Backend> FullPrecisionLinear<B> {
impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let batch_dim = state.active_row_count();
- let input_array = state.array(self.input_array_id);
- let output_array = state.array(self.output_array_id);
-
- let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.precision))?;
self.kernel.borrow_mut().encode(
- state.context(),
MatmulArguments {
- a: input_array.buffer().borrow().deref(),
+ a: &input,
a_offset: 0,
- b: self.weights_buffer.borrow().deref(),
+ b: &self.weights,
ab_scale: 1.0,
- c: match bias_borrow.as_deref() {
- Some(b) => MatmulArgumentC::Bias(b),
+ c: match self.bias.as_ref() {
+ Some(bias) => MatmulArgumentC::Bias(bias),
None => MatmulArgumentC::None,
},
- d: output_array.buffer().borrow_mut().deref_mut(),
+ d: &mut output,
batch_dim: batch_dim as u32,
input_dim: self.input_dim as u32,
output_dim: self.output_dim as u32,
},
encoder,
);
- Ok(())
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index 8124cb042..33a88f2ba 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -10,18 +10,18 @@ pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};
use thiserror::Error;
use crate::{
- backends::common::{Backend, Encoder},
+ backends::common::{Allocation, Backend, Encoder},
config::LinearConfig,
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
pub trait Linear<B: Backend> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error>;
+ ) -> Result<Allocation<B>, B::Error>;
}
#[derive(Debug, Error)]
@@ -30,10 +30,10 @@ pub enum LinearBlockError<B: Backend> {
QuantizedLinearError(#[from] QuantizedLinearError<B>),
#[error("FullPrecisionLinear error: {0}")]
FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),
- #[error("RHTLinearWrapper error: {0}")]
- RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),
#[error("QLoRALinearWrapper error: {0}")]
QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),
+ #[error("RHTLinearWrapper error: {0}")]
+ RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),
#[error("Parameter loading error: {0}")]
ParameterError(#[from] ParameterLoaderError<B>),
}
@@ -41,13 +41,10 @@ pub enum LinearBlockError<B: Backend> {
impl<B: Backend> dyn Linear<B> {
pub fn new<const N: usize>(
config: &LinearConfig,
- _has_biases: bool,
input_dimension: usize,
output_dimensions: [usize; N],
context: &B::Context,
parameter_tree: &ParameterTree<B::Context>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
@@ -58,8 +55,6 @@ impl<B: Backend> dyn Linear<B> {
input_dimension,
output_dimension_sum,
parameter_tree,
- input_array_id,
- output_array_id,
None,
)?;
Ok(Box::new(block))
@@ -73,8 +68,6 @@ impl<B: Backend> dyn Linear<B> {
input_dimension,
output_dimension_sum,
parameter_tree,
- input_array_id,
- output_array_id,
)?;
Ok(Box::new(block))
},
@@ -92,37 +85,28 @@ impl<B: Backend> dyn Linear<B> {
output_dimension_sum,
parameter_tree,
None,
- input_array_id,
- output_array_id,
)?;
Ok(Box::new(block))
},
LinearConfig::RHTLinearWrapper {
block_size,
inner_config,
- } => {
- let block = RHTLinearWrapper::new(
- context,
- *block_size,
- inner_config,
- input_dimension,
- output_dimension_sum,
- parameter_tree,
- input_array_id,
- output_array_id,
- )?;
- Ok(Box::new(block))
- },
+ } => Ok(Box::new(RHTLinearWrapper::new(
+ context,
+ *block_size,
+ inner_config,
+ input_dimension,
+ output_dimension_sum,
+ parameter_tree,
+ )?)),
}
}
pub fn new_with_output_hadamard(
context: &B::Context,
config: &LinearConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
- output_factors: B::DenseBuffer,
+ output_factors: Allocation<B>,
input_dim: usize,
output_dim: usize,
) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
@@ -133,8 +117,6 @@ impl<B: Backend> dyn Linear<B> {
input_dim,
output_dim,
parameter_tree,
- input_array_id,
- output_array_id,
Some(output_factors),
)?)),
LinearConfig::QLoRA {
@@ -150,8 +132,6 @@ impl<B: Backend> dyn Linear<B> {
output_dim,
parameter_tree,
Some(output_factors),
- input_array_id,
- output_array_id,
)?)),
inner_config => unimplemented!("{inner_config:?} doesn't support fused output hadamard"),
}
@@ -159,28 +139,23 @@ impl<B: Backend> dyn Linear<B> {
pub fn new_extracting_input_hadamard<const N: usize>(
config: &LinearConfig,
- _has_biases: bool,
input_dimension: usize,
output_dimensions: [usize; N],
context: &B::Context,
parameter_tree: &ParameterTree<B::Context>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- ) -> Result<(Box<dyn Linear<B>>, Option<B::DenseBuffer>), LinearBlockError<B>> {
+ ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
LinearConfig::RHTLinearWrapper {
inner_config,
..
} => {
- let input_factors = parameter_tree.leaf("input_factors")?.read_buffer()?;
- let output_factors = parameter_tree.leaf("output_factors")?.read_buffer()?;
+ let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;
+ let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;
let inner_tree = parameter_tree.subtree("inner_linear")?;
let inner_linear = Self::new_with_output_hadamard(
context,
inner_config,
- input_array_id,
- output_array_id,
&inner_tree,
output_factors,
input_dimension,
@@ -189,16 +164,7 @@ impl<B: Backend> dyn Linear<B> {
Ok((inner_linear, Some(input_factors)))
},
other => {
- let linear = Self::new(
- other,
- _has_biases,
- input_dimension,
- output_dimensions,
- context,
- parameter_tree,
- input_array_id,
- output_array_id,
- )?;
+ let linear = Self::new(other, input_dimension, output_dimensions, context, parameter_tree)?;
Ok((linear, None))
},
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
index 93fc77d24..380899244 100644
--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs
@@ -1,14 +1,12 @@
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
-};
+use std::cell::RefCell;
use thiserror::Error;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{
ManualKernels,
matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
@@ -19,7 +17,6 @@ use crate::{
Linear,
linear::{LinearBlockError, QuantizedLinear, QuantizedLinearError},
},
- forward_pass::state::{ArrayId, ForwardPassState},
prelude::{ParameterLeaf, ParameterLoaderError, ParameterTree},
};
@@ -45,14 +42,13 @@ pub enum QLoRALinearWrapperError<B: Backend> {
pub struct QLoRALinearWrapper<B: Backend> {
base_linear: QuantizedLinear<B>,
adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,
- adapter_down: B::DenseBuffer,
- adapter_up: B::DenseBuffer,
+ adapter_down: Allocation<B>,
+ adapter_up: Allocation<B>,
input_dim: usize,
output_dim: usize,
lora_rank: usize,
lora_scale: f32,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
+ data_type: DataType,
}
// TODO: figure out how to make this generic over QLoRAWrapperError::InvalidTensor or make one global "Invalid Tensor" error and make this a common helper
@@ -85,9 +81,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
input_dim: usize,
output_dim: usize,
parameter_tree: &ParameterTree<B::Context>,
- output_quantized_hadamard_factors: Option<B::DenseBuffer>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
+ output_quantized_hadamard_factors: Option<Allocation<B>>,
) -> Result<Self, QLoRALinearWrapperError<B>> {
let data_type = quantization.activation_precision.into();
@@ -97,8 +91,6 @@ impl<B: Backend> QLoRALinearWrapper<B> {
input_dim,
output_dim,
parameter_tree,
- input_array_id,
- output_array_id,
output_quantized_hadamard_factors,
)?;
let adapter_kernel =
@@ -106,11 +98,11 @@ impl<B: Backend> QLoRALinearWrapper<B> {
let adapter_down_leaf = parameter_tree.leaf("down_weights")?;
validate_tensor(&adapter_down_leaf, [lora_rank as usize, input_dim as usize], data_type)?;
- let adapter_down = adapter_down_leaf.read_buffer()?;
+ let adapter_down = adapter_down_leaf.read_allocation()?;
let adapter_up_leaf = parameter_tree.leaf("up_weights")?;
validate_tensor(&adapter_up_leaf, [output_dim, lora_rank as usize], data_type)?;
- let adapter_up = adapter_up_leaf.read_buffer()?;
+ let adapter_up = adapter_up_leaf.read_allocation()?;
Ok(Self {
base_linear,
@@ -121,8 +113,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {
output_dim,
lora_rank,
lora_scale,
- input_array_id,
- output_array_id,
+ data_type,
})
}
}
@@ -130,27 +121,22 @@ impl<B: Backend> QLoRALinearWrapper<B> {
impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), <B as Backend>::Error> {
- self.base_linear.encode(state, encoder)?;
-
+ ) -> Result<Allocation<B>, <B as Backend>::Error> {
let mut adapter_kernel = self.adapter_kernel.borrow_mut();
- let batch_dim = state.active_row_count();
-
- let intermediate_array = state.common_aux.lora_intermediate.as_ref().unwrap();
- let input_array = state.array(self.input_array_id);
- let output_array = state.array(self.output_array_id);
+ let mut intermediate =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.data_type))?;
adapter_kernel.encode(
- state.context(),
MatmulArguments {
- a: input_array.buffer().borrow().deref(),
+ a: &input,
a_offset: 0,
b: &self.adapter_down,
ab_scale: 1.0,
c: MatmulArgumentC::None,
- d: intermediate_array.buffer().borrow_mut().deref_mut(),
+ d: &mut intermediate,
batch_dim: batch_dim as u32,
input_dim: self.input_dim as u32,
output_dim: self.lora_rank as u32,
@@ -158,15 +144,16 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
encoder,
);
+ let mut output = self.base_linear.encode(input, batch_dim, encoder)?;
+
adapter_kernel.encode(
- state.context(),
MatmulArguments {
- a: intermediate_array.buffer().borrow().deref(),
+ a: &intermediate,
a_offset: 0,
b: &self.adapter_up,
ab_scale: self.lora_scale,
c: MatmulArgumentC::Accumulate,
- d: output_array.buffer().borrow_mut().deref_mut(),
+ d: &mut output,
batch_dim: batch_dim as u32,
input_dim: self.lora_rank as u32,
output_dim: self.output_dim as u32,
@@ -174,6 +161,6 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {
encoder,
);
- Ok(())
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/quantized.rs b/crates/backend-uzu/src/encodable_block/linear/quantized.rs
index 101cf36aa..4c1df9879 100644
--- a/crates/backend-uzu/src/encodable_block/linear/quantized.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/quantized.rs
@@ -1,16 +1,11 @@
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
use thiserror::Error;
use super::Linear;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{
Kernels, TensorAddBiasKernel,
quant_matmul::{
@@ -20,7 +15,6 @@ use crate::{
},
},
config::QuantizationConfig,
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -88,14 +82,13 @@ pub enum QuantizedLinearError<B: Backend> {
pub struct QuantizedLinear<B: Backend> {
kernel: QuantizedMatmulKernelEncodable<B>,
bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
- biases_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,
- weights_buffer: Rc<RefCell<B::DenseBuffer>>,
- scales_buffer: Rc<RefCell<B::DenseBuffer>>,
- zero_points_or_biases_buffer: Rc<RefCell<B::DenseBuffer>>,
- output_hadamard_factors: Option<B::DenseBuffer>,
+ biases: Option<Allocation<B>>,
+ weights: Allocation<B>,
+ scales: Allocation<B>,
+ zero_points_or_biases: Allocation<B>,
+ output_hadamard_factors: Option<Allocation<B>>,
output_dim: usize,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
+ output_data_type: DataType,
}
impl<B: Backend> QuantizedLinear<B> {
@@ -105,39 +98,37 @@ impl<B: Backend> QuantizedLinear<B> {
input_dim: usize,
output_dim: usize,
parameter_tree: &ParameterTree<B::Context>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- output_hadamard_factors: Option<B::DenseBuffer>,
+ output_hadamard_factors: Option<Allocation<B>>,
) -> Result<Self, QuantizedLinearError<B>> {
let kernel_data_type: DataType = config.activation_precision.into();
if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));
}
- let weights = parameter_tree.leaf_array("weights").map_err(QuantizedLinearError::ParameterError)?;
+ let weights_leaf = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;
let packing_divisor = config.weight_quantization_mode.packing_divisor();
let storage_type = config.weight_quantization_mode.storage_type();
- if weights.data_type() != storage_type {
+ if weights_leaf.data_type() != storage_type {
return Err(QuantizedLinearError::InvalidWeightsDataType {
expected: storage_type,
- got: weights.data_type(),
+ got: weights_leaf.data_type(),
});
}
- let scales = parameter_tree.leaf_array("scales").map_err(QuantizedLinearError::ParameterError)?;
- if scales.data_type() != kernel_data_type {
+ let scales_leaf = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;
+ if scales_leaf.data_type() != kernel_data_type {
return Err(QuantizedLinearError::InvalidScalesDataType {
expected: kernel_data_type,
- got: scales.data_type(),
+ got: scales_leaf.data_type(),
});
}
let k_g = (input_dim + config.group_size - 1) / config.group_size;
- let weights_shape = weights.shape().to_vec();
- let scales_shape = scales.shape().to_vec();
- let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf_array("deq_biases") {
- Ok(deq_biases) => {
- let deq_biases_shape = deq_biases.shape().to_vec();
+ let weights_shape = weights_leaf.shape().to_vec();
+ let scales_shape = scales_leaf.shape().to_vec();
+ let (quantization_type, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {
+ Ok(deq_biases_leaf) => {
+ let deq_biases_shape = deq_biases_leaf.shape().to_vec();
if !(weights_shape == [output_dim, input_dim / packing_divisor]
&& scales_shape == [output_dim, k_g]
&& deq_biases_shape == [output_dim, k_g])
@@ -150,19 +141,22 @@ impl<B: Backend> QuantizedLinear<B> {
});
}
- if deq_biases.data_type() != kernel_data_type {
+ if deq_biases_leaf.data_type() != kernel_data_type {
return Err(QuantizedLinearError::InvalidDeqBiasesDataType {
expected: kernel_data_type,
- got: deq_biases.data_type(),
+ got: deq_biases_leaf.data_type(),
});
}
- (QuantizedMatmulType::Mlx, deq_biases.buffer())
+ (
+ QuantizedMatmulType::Mlx,
+ deq_biases_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
+ )
},
Err(_) => {
- let zero_points =
- parameter_tree.leaf_array("zero_points").map_err(QuantizedLinearError::ParameterError)?;
- let zero_points_shape = zero_points.shape().to_vec();
+ let zero_points_leaf =
+ parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;
+ let zero_points_shape = zero_points_leaf.shape().to_vec();
let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;
if !(weights_shape == [output_dim, input_dim / packing_divisor]
&& scales_shape == [output_dim, k_g]
@@ -177,20 +171,23 @@ impl<B: Backend> QuantizedLinear<B> {
});
}
- if zero_points.data_type() != storage_type {
+ if zero_points_leaf.data_type() != storage_type {
return Err(QuantizedLinearError::InvalidZeroPointsDataType {
expected: storage_type,
- got: zero_points.data_type(),
+ got: zero_points_leaf.data_type(),
});
}
- (QuantizedMatmulType::ZeroPoint, zero_points.buffer())
+ (
+ QuantizedMatmulType::ZeroPoint,
+ zero_points_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
+ )
},
};
- let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf_array("biases") {
- Ok(biases) => {
- let bias_shape = biases.shape().to_vec();
+ let (bias_add_kernel, biases) = match parameter_tree.leaf("biases") {
+ Ok(biases_leaf) => {
+ let bias_shape = biases_leaf.shape().to_vec();
if bias_shape != [output_dim] {
return Err(QuantizedLinearError::InvalidBiasShape {
got: bias_shape.into_boxed_slice(),
@@ -198,17 +195,20 @@ impl<B: Backend> QuantizedLinear<B> {
});
}
- if biases.data_type() != kernel_data_type {
+ if biases_leaf.data_type() != kernel_data_type {
return Err(QuantizedLinearError::InvalidBiasDataType {
expected: kernel_data_type,
- got: biases.data_type(),
+ got: biases_leaf.data_type(),
});
}
let bias_add_kernel =
<B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type, true)
.map_err(QuantizedLinearError::BackendError)?;
- (Some(bias_add_kernel), Some(biases.buffer()))
+ (
+ Some(bias_add_kernel),
+ Some(biases_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?),
+ )
},
Err(_) => (None, None),
};
@@ -230,14 +230,13 @@ impl<B: Backend> QuantizedLinear<B> {
Ok(Self {
kernel,
bias_add_kernel,
- biases_buffer,
- weights_buffer: weights.buffer(),
- scales_buffer: scales.buffer(),
- zero_points_or_biases_buffer,
+ biases,
+ weights: weights_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
+ scales: scales_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,
+ zero_points_or_biases,
output_hadamard_factors,
output_dim,
- input_array_id,
- output_array_id,
+ output_data_type: kernel_data_type,
})
}
}
@@ -245,43 +244,39 @@ impl<B: Backend> QuantizedLinear<B> {
impl<B: Backend> Linear<B> for QuantizedLinear<B> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let batch_dim = state.active_row_count();
-
- let input_array = state.array(self.input_array_id);
- let output_array = state.array(self.output_array_id);
- let output_buf_rc = output_array.buffer();
- let mut output_buf_borrow = output_buf_rc.borrow_mut();
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut output =
+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;
- self.kernel
- .encode(
- encoder,
- QuantizedMatmulArguments {
- a_buffer: input_array.buffer().borrow().deref(),
- a_offset: 0,
- b_buffer: self.weights_buffer.borrow().deref(),
- scales_buffer: self.scales_buffer.borrow().deref(),
- zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),
- output_buffer: output_buf_borrow.deref_mut(),
- hadamard_factors: self.output_hadamard_factors.as_ref(),
- batch_dim,
- },
- )
- .expect("Failed to encode quantized matmul");
+ self.kernel.encode(
+ encoder,
+ QuantizedMatmulArguments {
+ a: &input,
+ a_offset: 0,
+ b: &self.weights,
+ scales: &self.scales,
+ zero_points_or_biases: &self.zero_points_or_biases,
+ output: &mut output,
+ hadamard_factors: self.output_hadamard_factors.as_ref(),
+ batch_dim,
+ },
+ );
- if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {
+ if let (Some(bias_add_kernel), Some(biases)) = (&self.bias_add_kernel, &self.biases) {
let total_length = batch_dim * self.output_dim;
bias_add_kernel.encode(
- None::<&B::DenseBuffer>,
- biases_buffer.borrow().deref(),
- output_buf_borrow.deref_mut(),
+ None::<&Allocation<B>>,
+ biases,
+ &mut output,
self.output_dim as u32,
total_length as u32,
encoder,
);
}
- Ok(())
+
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
index f15033b35..2714551f4 100644
--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs
@@ -1,15 +1,12 @@
-use std::ops::DerefMut;
-
use thiserror::Error;
use super::{Linear, LinearBlockError};
use crate::{
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{HadamardTransformKernel, Kernels},
},
config::LinearConfig,
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -25,10 +22,9 @@ pub enum RHTLinearWrapperError<B: Backend> {
pub struct RHTLinearWrapper<B: Backend> {
input_hadamard_kernel: <B::Kernels as Kernels>::HadamardTransformKernel,
- input_factors: B::DenseBuffer,
+ input_factors: Allocation<B>,
inner_linear: Box<dyn Linear<B>>,
input_dimension: usize,
- input_array_id: ArrayId,
}
impl<B: Backend> RHTLinearWrapper<B> {
@@ -39,42 +35,30 @@ impl<B: Backend> RHTLinearWrapper<B> {
input_dimension: usize,
output_dimension: usize,
parameter_tree: &ParameterTree<B::Context>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
) -> Result<Self, RHTLinearWrapperError<B>> {
assert!(block_size == 32, "only block size 32 hadamard is supported");
let data_type = inner_config.activation_precision().into();
-
- let input_factors_leaf = parameter_tree.leaf("input_factors")?;
- let output_factors_leaf = parameter_tree.leaf("output_factors")?;
-
- let input_factors = input_factors_leaf.read_buffer()?;
- let output_factors = output_factors_leaf.read_buffer()?;
-
+ let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;
+ let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;
let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type)
.map_err(RHTLinearWrapperError::BackendError)?;
-
let inner_linear_tree = parameter_tree.subtree("inner_linear")?;
-
let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(
context,
inner_config,
- input_array_id,
- output_array_id,
&inner_linear_tree,
output_factors,
input_dimension,
output_dimension,
)
- .map_err(|e| RHTLinearWrapperError::InnerLinearError(Box::new(e)))?;
+ .map_err(|error| RHTLinearWrapperError::InnerLinearError(Box::new(error)))?;
Ok(Self {
input_hadamard_kernel,
input_factors,
inner_linear,
input_dimension,
- input_array_id,
})
}
}
@@ -82,20 +66,17 @@ impl<B: Backend> RHTLinearWrapper<B> {
impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ mut input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- // Input Hadamard
- let input_array = state.array(self.input_array_id);
+ ) -> Result<Allocation<B>, B::Error> {
self.input_hadamard_kernel.encode(
- input_array.buffer().borrow_mut().deref_mut(),
+ &mut input,
&self.input_factors,
self.input_dimension as u32,
- state.active_row_count() as u32,
+ batch_dim as u32,
encoder,
);
-
- // Matmul with fused output Hadamard
- self.inner_linear.encode(state, encoder)
+ self.inner_linear.encode(input, batch_dim, encoder)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
index 361074d9c..b2b7bb66a 100644
--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs
@@ -1,28 +1,38 @@
//! Mamba2 SSM mixer encodable.
-use std::{
- env,
- ops::{Deref, DerefMut},
-};
+use std::env;
+
+use thiserror::Error;
use crate::{
DataType,
- array::Array,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{
Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,
ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
config::{DecoderLayerType, Mamba2Config},
- encodable_block::linear::Linear,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::{ParameterTree, resolve_subtree},
+ encodable_block::linear::{Linear, LinearBlockError},
+ forward_pass::ssm_layer::SSMLayer,
+ parameters::{ParameterLoaderError, ParameterTree, try_resolve_subtree},
};
+#[derive(Debug, Error)]
+pub enum MambaMixerError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Unsupported configuration: {0}")]
+ UnsupportedConfiguration(String),
+ #[error("Linear error: {0}")]
+ LinearError(#[from] Box<LinearBlockError<B>>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+}
+
pub(crate) struct MambaMixer<B: Backend> {
- layer_index: usize,
config: Mamba2Config,
in_projection: Box<dyn Linear<B>>,
out_projection: Box<dyn Linear<B>>,
@@ -32,11 +42,29 @@ pub(crate) struct MambaMixer<B: Backend> {
conv_scan: <B::Kernels as Kernels>::Conv1dScanKernel,
ssd_prefill: SSDPrefillKernels<B>,
ssd_update: <B::Kernels as Kernels>::SSDUpdateKernel,
- conv_weight: Array<B>,
- conv_bias: Option<Array<B>>,
- gate_bias: Array<B>,
- skip_connection_weight: Array<B>,
+ conv_weight: Allocation<B>,
+ conv_bias: Option<Allocation<B>>,
+ gate_bias: Allocation<B>,
+ skip_connection_weight: Allocation<B>,
prefill_mode: SSDPrefillMode,
+ data_type: DataType,
+}
+
+pub(crate) struct MambaArguments<'a, B: Backend> {
+ pub active_row_count: usize,
+ pub layer: &'a mut SSMLayer<B>,
+}
+
+struct SplitInProjectionOutput<B: Backend> {
+ conv_inputs: Allocation<B>,
+ gate: Allocation<B>,
+ time_step: Allocation<B>,
+}
+
+struct ConvScanOutput<B: Backend> {
+ conv_x: Allocation<B>,
+ state_b: Allocation<B>,
+ state_c: Allocation<B>,
}
fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
@@ -58,155 +86,151 @@ impl<B: Backend> MambaMixer<B> {
mamba_config: Mamba2Config,
layer_index: usize,
model_dim: usize,
- num_heads: usize,
- head_dim: usize,
- num_groups: usize,
decoder_layer_loader: &ParameterTree<B::Context>,
- ) -> Self {
- let _ = (num_heads, head_dim, num_groups);
+ ) -> Result<(Self, Option<Allocation<B>>), MambaMixerError<B>> {
if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {
- panic!("Layer {} marked as transformer but Mamba mixer config provided", layer_index);
+ return Err(MambaMixerError::UnsupportedConfiguration(format!(
+ "layer {layer_index} marked as transformer but Mamba mixer config provided"
+ )));
}
- let split_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
- let conv_tree = resolve_subtree(&split_tree, &["conv", "conv1d"]);
+ let split_tree = try_resolve_subtree(decoder_layer_loader, &["mixer"])?;
+ let conv_tree = try_resolve_subtree(&split_tree, &["conv", "conv1d"])?;
let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();
- let in_projection = <dyn Linear<B>>::new(
+ let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&mamba_config.in_projection_config,
- mamba_config.has_in_biases,
model_dim,
[mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],
context,
- &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),
- ArrayId::Main,
- ArrayId::SsmInProj,
+ &try_resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"])?,
)
- .expect("Failed to create in-projection kernel");
+ .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;
let out_projection = <dyn Linear<B>>::new(
&mamba_config.out_projection_config,
- mamba_config.has_out_biases,
mamba_config.inner_dim(),
[model_dim],
context,
- &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),
- ArrayId::AttentionOutput,
- ArrayId::Main,
+ &try_resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"])?,
)
- .expect("Failed to create out-projection kernel");
+ .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;
- let conv_weight = conv_tree.leaf_array("weights").unwrap();
+ let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;
let conv_bias = if mamba_config.conv_config.has_biases {
- Some(conv_tree.leaf_array("biases").unwrap())
+ Some(conv_tree.leaf("biases")?.read_allocation()?)
} else {
None
};
- let gate_bias = split_tree.leaf_array("gate_bias").unwrap();
- let skip_connection_weight = split_tree.leaf_array("skip_connection_weight").unwrap();
+ let gate_bias = split_tree.leaf("gate_bias")?.read_allocation()?;
+ let skip_connection_weight = split_tree.leaf("skip_connection_weight")?.read_allocation()?;
let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)
- .expect("Failed to create split in-projection kernel");
+ .map_err(MambaMixerError::BackendError)?;
let conv_scan =
<B::Kernels as Kernels>::Conv1dScanKernel::new(context, data_type, mamba_config.conv_config.has_biases)
- .expect("Failed to create conv scan kernel");
+ .map_err(MambaMixerError::BackendError)?;
let conv_decode = <B::Kernels as Kernels>::Conv1dDecodeKernel::new(
context,
data_type,
mamba_config.conv_config.has_biases,
true,
)
- .expect("Failed to create conv decode kernel");
+ .map_err(MambaMixerError::BackendError)?;
let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)
- .expect("Failed to create conv pack kernel");
- let ssd_prefill = SSDPrefillKernels::new(context, data_type).expect("Failed to create SSD prefill kernel");
+ .map_err(MambaMixerError::BackendError)?;
+ let ssd_prefill = SSDPrefillKernels::new(context, data_type).map_err(MambaMixerError::BackendError)?;
let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type, true)
- .expect("Failed to create SSD decode kernel");
+ .map_err(MambaMixerError::BackendError)?;
let prefill_mode = resolve_prefill_mode_from_env();
- Self {
- layer_index,
- config: mamba_config,
- in_projection,
- out_projection,
- split_inproj,
- conv_decode,
- conv_pack,
- conv_scan,
- ssd_prefill,
- ssd_update,
- conv_weight,
- conv_bias,
- gate_bias,
- skip_connection_weight,
- prefill_mode,
- }
+ Ok((
+ Self {
+ config: mamba_config,
+ in_projection,
+ out_projection,
+ split_inproj,
+ conv_decode,
+ conv_pack,
+ conv_scan,
+ ssd_prefill,
+ ssd_update,
+ conv_weight,
+ conv_bias,
+ gate_bias,
+ skip_connection_weight,
+ prefill_mode,
+ data_type,
+ },
+ in_projection_input_hadamard_factors,
+ ))
}
fn run_split_inproj(
&self,
- state: &mut ForwardPassState<B>,
- encoder: &mut Encoder<B>,
+ in_proj: &Allocation<B>,
suffix_length: usize,
- ) {
- let in_proj = state.array(ArrayId::SsmInProj);
- let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));
- let gate = state.array(ArrayId::SsmZ(self.layer_index));
- let dt = state.array(ArrayId::SsmDt(self.layer_index));
-
+ encoder: &mut Encoder<B>,
+ ) -> Result<SplitInProjectionOutput<B>, B::Error> {
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let num_heads = self.config.num_heads;
let total_dim = conv_dim + inner_dim + num_heads;
-
+ let mut conv_inputs = encoder.allocate_scratch(size_for_shape(&[suffix_length, conv_dim], self.data_type))?;
+ let mut gate = encoder
+ .allocate_scratch(size_for_shape(&[suffix_length, num_heads, self.config.head_dim], self.data_type))?;
+ let mut time_step = encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads], self.data_type))?;
self.split_inproj.encode(
- in_proj.buffer().borrow().deref(),
- conv_inputs.buffer().borrow_mut().deref_mut(),
- gate.buffer().borrow_mut().deref_mut(),
- dt.buffer().borrow_mut().deref_mut(),
- self.gate_bias.buffer().borrow().deref(),
+ in_proj,
+ &mut conv_inputs,
+ &mut gate,
+ &mut time_step,
+ &self.gate_bias,
suffix_length as u32,
total_dim as u32,
conv_dim as u32,
inner_dim as u32,
num_heads as u32,
encoder,
- )
+ );
+ Ok(SplitInProjectionOutput {
+ conv_inputs,
+ gate,
+ time_step,
+ })
}
fn run_conv_scan(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut SSMLayer<B>,
+ conv_inputs: &Allocation<B>,
encoder: &mut Encoder<B>,
suffix_length: usize,
- ) {
- let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));
- let conv_state = state.array(ArrayId::SsmConvState(self.layer_index));
- let x_arr = state.array(ArrayId::SsmX(self.layer_index));
- let b_arr = state.array(ArrayId::SsmB(self.layer_index));
- let c_arr = state.array(ArrayId::SsmC(self.layer_index));
-
- let weight_buf_rc = self.conv_weight.buffer();
- let weight_buf_borrow = weight_buf_rc.borrow();
- let bias_buf_rc = self.conv_bias.as_ref().map(|arr| arr.buffer());
- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
-
+ ) -> Result<ConvScanOutput<B>, B::Error> {
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let proj_dim = self.config.num_groups * self.config.state_dim;
let state_stride = self.config.kernel_size.saturating_sub(1);
+ let mut conv_x = encoder.allocate_scratch(size_for_shape(
+ &[suffix_length, self.config.num_heads, self.config.head_dim],
+ self.data_type,
+ ))?;
+ let state_shape = [suffix_length, self.config.num_groups, self.config.state_dim];
+ let mut state_b = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;
+ let mut state_c = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;
if suffix_length == 1 {
if conv_dim > 0 && self.config.kernel_size > 0 {
+ let next_state = layer.conv_state.as_mut().unwrap_or(&mut layer.ssm_state);
self.conv_decode.encode(
- conv_inputs.buffer().borrow().deref(),
- weight_buf_borrow.deref(),
- bias_buf_borrow.as_deref(),
- None::<&B::DenseBuffer>,
- x_arr.buffer().borrow_mut().deref_mut(),
- b_arr.buffer().borrow_mut().deref_mut(),
- c_arr.buffer().borrow_mut().deref_mut(),
- conv_state.buffer().borrow_mut().deref_mut(),
+ conv_inputs,
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ None::<&Allocation<B>>,
+ &mut conv_x,
+ &mut state_b,
+ &mut state_c,
+ next_state,
self.config.kernel_size as u32,
conv_dim as u32,
state_stride as u32,
@@ -219,39 +243,39 @@ impl<B: Backend> MambaMixer<B> {
);
}
} else {
- let padded_buf = if state_stride > 0 {
- let array = state.conv_padded_buffer().expect("Missing conv padded buffer");
- let buffer = array.buffer();
-
+ let mut padded = (conv_dim > 0 && state_stride > 0)
+ .then(|| {
+ encoder.allocate_scratch(size_for_shape(&[suffix_length + state_stride, conv_dim], self.data_type))
+ })
+ .transpose()?;
+ if let Some(padded) = padded.as_mut() {
+ let state_in = layer.conv_state.as_ref().unwrap_or(conv_inputs);
self.conv_pack.encode(
- conv_state.buffer().borrow().deref(),
- conv_inputs.buffer().borrow().deref(),
- buffer.borrow_mut().deref_mut(),
+ state_in,
+ conv_inputs,
+ padded,
state_stride as u32,
conv_dim as u32,
suffix_length as u32,
conv_dim as u32,
encoder,
);
-
- Some(buffer)
- } else {
- None
- };
+ }
if conv_dim > 0 && self.config.kernel_size > 0 {
- let padded_borrow = padded_buf.as_ref().map(|b| b.borrow());
- let conv_inputs_borrow = conv_inputs.buffer();
- let conv_inputs_ref = conv_inputs_borrow.borrow();
- let conv_source: &B::DenseBuffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());
+ let conv_source = match &padded {
+ Some(padded) => padded,
+ None => conv_inputs,
+ };
+ let next_state = layer.conv_state.as_mut().unwrap_or(&mut layer.ssm_state);
self.conv_scan.encode(
conv_source,
- weight_buf_borrow.deref(),
- bias_buf_borrow.as_deref(),
- x_arr.buffer().borrow_mut().deref_mut(),
- b_arr.buffer().borrow_mut().deref_mut(),
- c_arr.buffer().borrow_mut().deref_mut(),
- conv_state.buffer().borrow_mut().deref_mut(),
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ &mut conv_x,
+ &mut state_b,
+ &mut state_c,
+ next_state,
suffix_length as u32,
self.config.kernel_size as u32,
conv_dim as u32,
@@ -261,36 +285,40 @@ impl<B: Backend> MambaMixer<B> {
proj_dim as u32,
self.config.activation.act_type(),
encoder,
- )
+ );
}
}
+ Ok(ConvScanOutput {
+ conv_x,
+ state_b,
+ state_c,
+ })
}
fn run_prefill_ssm(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut SSMLayer<B>,
+ conv_x: &Allocation<B>,
+ state_b: &Allocation<B>,
+ state_c: &Allocation<B>,
+ time_step: &Allocation<B>,
+ gate: &Allocation<B>,
encoder: &mut Encoder<B>,
suffix_length: usize,
- ) {
- let x = state.array(ArrayId::SsmX(self.layer_index));
- let b = state.array(ArrayId::SsmB(self.layer_index));
- let c = state.array(ArrayId::SsmC(self.layer_index));
- let dt = state.array(ArrayId::SsmDt(self.layer_index));
- let z = state.array(ArrayId::SsmZ(self.layer_index));
- let state_arr = state.array(ArrayId::SsmState(self.layer_index));
- let out = state.array(ArrayId::AttentionOutput);
-
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut out =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;
self.ssd_prefill.encode(
encoder,
SSDPrefillArguments {
- x: x.buffer().borrow().deref(),
- dt: dt.buffer().borrow().deref(),
- b: b.buffer().borrow().deref(),
- c: c.buffer().borrow().deref(),
- d: self.skip_connection_weight.buffer().borrow().deref(),
- z: z.buffer().borrow().deref(),
- state: state_arr.buffer().borrow_mut().deref_mut(),
- y: out.buffer().borrow_mut().deref_mut(),
+ x: conv_x,
+ dt: time_step,
+ b: state_b,
+ c: state_c,
+ d: &self.skip_connection_weight,
+ z: gate,
+ state: &mut layer.ssm_state,
+ y: &mut out,
suffix_len: suffix_length,
group_size: (self.config.num_heads / self.config.num_groups) as u32,
state_size: self.config.state_dim as u32,
@@ -302,23 +330,23 @@ impl<B: Backend> MambaMixer<B> {
head_dim: self.config.head_dim,
},
self.prefill_mode,
- )
+ );
+ Ok(out)
}
fn run_decode_ssm(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut SSMLayer<B>,
+ conv_x: &Allocation<B>,
+ state_b: &Allocation<B>,
+ state_c: &Allocation<B>,
+ time_step: &Allocation<B>,
+ gate: &Allocation<B>,
encoder: &mut Encoder<B>,
suffix_length: usize,
- ) {
- let x = state.array(ArrayId::SsmX(self.layer_index));
- let b = state.array(ArrayId::SsmB(self.layer_index));
- let c = state.array(ArrayId::SsmC(self.layer_index));
- let dt = state.array(ArrayId::SsmDt(self.layer_index));
- let z = state.array(ArrayId::SsmZ(self.layer_index));
- let state_arr = state.array(ArrayId::SsmState(self.layer_index));
- let y = state.array(ArrayId::AttentionOutput);
-
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut out =
+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;
let h = self.config.num_heads as u32;
let g = self.config.num_groups as u32;
let dh = self.config.head_dim as u32;
@@ -332,15 +360,15 @@ impl<B: Backend> MambaMixer<B> {
let state_size = n as i32;
self.ssd_update.encode(
- x.buffer().borrow().deref(),
- dt.buffer().borrow().deref(),
- b.buffer().borrow().deref(),
- c.buffer().borrow().deref(),
- self.skip_connection_weight.buffer().borrow().deref(),
- z.buffer().borrow().deref(),
- None::<&B::DenseBuffer>,
- y.buffer().borrow_mut().deref_mut(),
- state_arr.buffer().borrow_mut().deref_mut(),
+ conv_x,
+ time_step,
+ state_b,
+ state_c,
+ &self.skip_connection_weight,
+ gate,
+ None::<&Allocation<B>>,
+ &mut out,
+ &mut layer.ssm_state,
group_size as u32,
state_size as u32,
&x_strides,
@@ -352,27 +380,40 @@ impl<B: Backend> MambaMixer<B> {
dh,
encoder,
);
+ Ok(out)
}
pub(crate) fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ args: MambaArguments<B>,
+ input: Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let active_row_count = state.active_row_count();
- if active_row_count == 0 {
- return Ok(());
- }
+ ) -> Result<Allocation<B>, B::Error> {
+ let MambaArguments {
+ active_row_count,
+ layer,
+ } = args;
+ assert!(active_row_count > 0, "Mamba mixer requires at least one active row");
- self.in_projection.encode(state, encoder)?;
- self.run_split_inproj(state, encoder, active_row_count);
- self.run_conv_scan(state, encoder, active_row_count);
+ let in_proj = self.in_projection.encode(input, active_row_count, encoder)?;
+ let SplitInProjectionOutput {
+ conv_inputs,
+ gate,
+ time_step,
+ } = self.run_split_inproj(&in_proj, active_row_count, encoder)?;
+ let ConvScanOutput {
+ conv_x,
+ state_b,
+ state_c,
+ } = self.run_conv_scan(layer, &conv_inputs, encoder, active_row_count)?;
if active_row_count == 1 {
- self.run_decode_ssm(state, encoder, active_row_count);
+ let ssm_output =
+ self.run_decode_ssm(layer, &conv_x, &state_b, &state_c, &time_step, &gate, encoder, active_row_count)?;
+ self.out_projection.encode(ssm_output, active_row_count, encoder)
} else {
- self.run_prefill_ssm(state, encoder, active_row_count);
+ let ssm_output =
+ self.run_prefill_ssm(layer, &conv_x, &state_b, &state_c, &time_step, &gate, encoder, active_row_count)?;
+ self.out_projection.encode(ssm_output, active_row_count, encoder)
}
- self.out_projection.encode(state, encoder)?;
- Ok(())
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
index 2d6558228..00449dea4 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs
@@ -1,17 +1,18 @@
//! MLP block encodable.
-use std::ops::{Deref, DerefMut};
-
use super::{super::linear::Linear, Mlp};
use crate::{
- backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
- forward_pass::state::{ArrayId, ForwardPassState},
+ DataType,
+ array::size_for_shape,
+ backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
};
pub struct DenseMlp<B: Backend> {
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
+ hidden_dim: usize,
+ data_type: DataType,
}
impl<B: Backend> DenseMlp<B> {
@@ -19,11 +20,15 @@ impl<B: Backend> DenseMlp<B> {
up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
down: Box<dyn Linear<B>>,
+ hidden_dim: usize,
+ data_type: DataType,
) -> Self {
Self {
up,
gate,
down,
+ hidden_dim,
+ data_type,
}
}
}
@@ -31,22 +36,13 @@ impl<B: Backend> DenseMlp<B> {
impl<B: Backend> Mlp<B> for DenseMlp<B> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- // Up
- self.up.encode(state, encoder)?;
-
- // Gate act+mul (fused_up -> hidden)
- {
- let fused = state.array(ArrayId::MlpFusedUp);
- let hidden = state.array(ArrayId::MlpHidden);
- let m = fused.shape()[0] as i32;
- self.gate.encode(encoder, fused.buffer().borrow().deref(), hidden.buffer().borrow_mut().deref_mut(), m)?;
- }
-
- // Down
- self.down.encode(state, encoder)?;
- Ok(())
+ ) -> Result<Allocation<B>, B::Error> {
+ let fused_up = self.up.encode(input, batch_dim, encoder)?;
+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;
+ self.gate.encode(encoder, &fused_up, &mut hidden, batch_dim as i32)?;
+ self.down.encode(hidden, batch_dim, encoder)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
index 61e91adf0..b301563c3 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs
@@ -8,18 +8,18 @@ use thiserror::Error;
use super::linear::{Linear, LinearBlockError};
use crate::{
DataType,
- backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
+ backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
config::MLPConfig,
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
pub trait Mlp<B: Backend> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error>;
+ ) -> Result<Allocation<B>, B::Error>;
}
#[derive(Debug, Error)]
@@ -41,30 +41,24 @@ impl<B: Backend> dyn Mlp<B> {
hidden_dimension: usize,
context: &B::Context,
parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<(Box<dyn Mlp<B>>, Option<B::DenseBuffer>), MlpBlockError<B>> {
+ ) -> Result<(Box<dyn Mlp<B>>, Option<Allocation<B>>), MlpBlockError<B>> {
if let MLPConfig::Dense(dense_config) = config {
let data_type: DataType = dense_config.linear_config.activation_precision().into();
let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&dense_config.linear_config,
- false,
model_dimension,
[2 * hidden_dimension],
context,
&parameter_tree.subtree("up_projection")?,
- ArrayId::Main,
- ArrayId::MlpFusedUp,
)?;
let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&dense_config.linear_config,
- false,
hidden_dimension,
[model_dimension],
context,
&parameter_tree.subtree("down_projection")?,
- ArrayId::MlpHidden,
- ArrayId::Main,
)?;
let gate = MlpGateActMulEncodable::new(
@@ -76,7 +70,10 @@ impl<B: Backend> dyn Mlp<B> {
)
.map_err(MlpBlockError::BackendError)?;
- return Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors));
+ return Ok((
+ Box::new(DenseMlp::new(up_projection, gate, down_projection, hidden_dimension, data_type)),
+ up_input_hadamard_factors,
+ ));
}
if let MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe.rs b/crates/backend-uzu/src/encodable_block/mlp/moe.rs
index ef9f50c49..c85d49635 100644
--- a/crates/backend-uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/backend-uzu/src/encodable_block/mlp/moe.rs
@@ -1,17 +1,12 @@
//! MoE (Mixture of Experts) block encodable.
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
use thiserror::Error;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
gpu_types::ActivationType,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
@@ -24,21 +19,19 @@ use crate::{
},
config::{LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig},
encodable_block::mlp::Mlp,
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
struct RouterBlock<B: Backend> {
- weights_buf: Rc<RefCell<B::DenseBuffer>>,
- biases_buf: Rc<RefCell<B::DenseBuffer>>,
+ weights: Allocation<B>,
+ biases: Allocation<B>,
}
-#[derive(Clone)]
struct SharedMoeWeights<B: Backend> {
- pub w13_buf: Rc<RefCell<B::DenseBuffer>>,
- pub w2_buf: Rc<RefCell<B::DenseBuffer>>,
- pub up_biases_buf: Rc<RefCell<B::DenseBuffer>>,
- pub down_biases_buf: Rc<RefCell<B::DenseBuffer>>,
+ pub w13: Allocation<B>,
+ pub w2: Allocation<B>,
+ pub up_biases: Allocation<B>,
+ pub down_biases: Allocation<B>,
}
pub struct MoeBlock<B: Backend> {
@@ -64,7 +57,9 @@ pub enum MoeBlockError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Parameter loader error: {0}")]
- ParameterLoaderError(#[source] ParameterLoaderError<B>),
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+ #[error("MoE num_active_routed_experts must be greater than 0")]
+ InvalidActiveExpertCount,
}
impl<B: Backend> MoeBlock<B> {
@@ -75,6 +70,9 @@ impl<B: Backend> MoeBlock<B> {
hidden_dim: usize,
parameter_tree: &ParameterTree<B::Context>,
) -> Result<Self, MoeBlockError<B>> {
+ if moe_config.num_active_routed_experts == 0 {
+ return Err(MoeBlockError::InvalidActiveExpertCount);
+ }
let data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();
let router_data_type: DataType = moe_config.router_config.activation_precision().into();
@@ -93,11 +91,13 @@ impl<B: Backend> MoeBlock<B> {
LinearConfig::FullPrecision {
..
} => {
- let weights_arr = router_tree.leaf_array("weights").map_err(MoeBlockError::ParameterLoaderError)?;
- let biases_arr = router_tree.leaf_array("biases").map_err(MoeBlockError::ParameterLoaderError)?;
+ let weights =
+ router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?.read_allocation()?;
+ let biases =
+ router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?.read_allocation()?;
RouterBlock {
- weights_buf: weights_arr.buffer(),
- biases_buf: biases_arr.buffer(),
+ weights,
+ biases,
}
},
LinearConfig::QLoRA {
@@ -131,35 +131,39 @@ impl<B: Backend> MoeBlock<B> {
let experts_tree = parameter_tree.subtree("experts").map_err(MoeBlockError::ParameterLoaderError)?;
- let w13_arr = experts_tree
+ let w13 = experts_tree
.subtree("up_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf_array("weights")
- .map_err(MoeBlockError::ParameterLoaderError)?;
+ .leaf("weights")
+ .map_err(MoeBlockError::ParameterLoaderError)?
+ .read_allocation()?;
- let w2_arr = experts_tree
+ let w2 = experts_tree
.subtree("down_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf_array("weights")
- .map_err(MoeBlockError::ParameterLoaderError)?;
+ .leaf("weights")
+ .map_err(MoeBlockError::ParameterLoaderError)?
+ .read_allocation()?;
- let up_biases_arr = experts_tree
+ let up_biases = experts_tree
.subtree("up_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf_array("biases")
- .map_err(MoeBlockError::ParameterLoaderError)?;
+ .leaf("biases")
+ .map_err(MoeBlockError::ParameterLoaderError)?
+ .read_allocation()?;
- let down_biases_arr = experts_tree
+ let down_biases = experts_tree
.subtree("down_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf_array("biases")
- .map_err(MoeBlockError::ParameterLoaderError)?;
+ .leaf("biases")
+ .map_err(MoeBlockError::ParameterLoaderError)?
+ .read_allocation()?;
let shared_weights = SharedMoeWeights {
- w13_buf: w13_arr.buffer(),
- w2_buf: w2_arr.buffer(),
- up_biases_buf: up_biases_arr.buffer(),
- down_biases_buf: down_biases_arr.buffer(),
+ w13,
+ w2,
+ up_biases,
+ down_biases,
};
Ok(Self {
@@ -197,68 +201,21 @@ impl<B: Backend> MoeBlock<B> {
impl<B: Backend> Mlp<B> for MoeBlock<B> {
fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let suffix_length = state.active_row_count();
-
- let main_buf = state.array(ArrayId::Main).buffer();
- let topk_ids_buf = state.array(ArrayId::MoeTopkIds).buffer();
- let topk_probs_buf = state.array(ArrayId::MoeTopkProbs).buffer();
- let offsets_buf = state.array(ArrayId::MoeOffsets).buffer();
- let sumk_buf = state.array(ArrayId::MoeSumK).buffer();
- let bucketed_ids_buf = state.array(ArrayId::MoeBucketedTokenIds).buffer();
- let bucketed_probs_buf = state.array(ArrayId::MoeBucketedProbs).buffer();
- let x_perm_buf = state.array(ArrayId::MoeXPerm).buffer();
- let tok2row_buf = state.array(ArrayId::MoeTok2Row).buffer();
- let y_partial_buf = state.array(ArrayId::MoeYPartial).buffer();
- let tile_counts_buf = state.array(ArrayId::MoeTileCounts).buffer();
- let tile_offsets_buf = state.array(ArrayId::MoeTileOffsets).buffer();
- let tile_map_buf = state.array(ArrayId::MoeTileMap).buffer();
- let total_tiles_buf = state.array(ArrayId::MoeTotalTiles).buffer();
- let dispatch_args_buf = state.array(ArrayId::MoeDispatchArgs).buffer();
- let partials_buf = state.array(ArrayId::MoeScatterPartials).buffer();
- let block_bases_buf = state.array(ArrayId::MoeScatterBlockBases).buffer();
- let block_alloc_buf = state.array(ArrayId::MoeBlockAlloc).buffer();
- let hidden_buf = state.array(ArrayId::MoeHidden).buffer();
- let row_expert_map_buf = state.array(ArrayId::MoeTwoPassRowExpertMap).buffer();
+ ) -> Result<Allocation<B>, B::Error> {
+ let suffix_length = batch_dim;
+ if suffix_length == 0 {
+ return Ok(input);
+ }
let e = self.moe_config.num_routed_experts;
let k = self.moe_config.num_active_routed_experts;
let k_tile = 128;
-
- // Clear internal MoE buffers
- if suffix_length > 0 && k > 0 {
- let entries = suffix_length * k;
- let topk_bytes = entries * std::mem::size_of::<u32>();
- let tok2row_bytes = entries * std::mem::size_of::<i32>();
-
- // Clear topk_ids and tok2row buffers
- if topk_bytes > 0 {
- encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);
- }
- if tok2row_bytes > 0 {
- encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);
- }
-
- // Clear hidden buffer
- let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
- if hidden_bytes > 0 {
- encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);
- }
-
- // Clear y_partial buffer
- let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
- if y_partial_bytes > 0 {
- encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);
- }
-
- // Clear x_perm buffer
- let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
- if x_perm_bytes > 0 {
- encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);
- }
- }
+ let total_rows = suffix_length * k;
+ let num_blocks = suffix_length.div_ceil(256).max(1);
+ let num_tiles = e.div_ceil(512).max(1);
// Use fused Router+TopK kernel for non-quantized routers
if self.model_dim % 4 != 0 {
@@ -271,14 +228,20 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
panic!("MoE fused router+bottomk failed: {k} > 128");
}
+ let mut topk_ids = encoder.allocate_scratch(size_for_shape(&[suffix_length, k], DataType::I32))?;
+ let mut topk_probs = encoder.allocate_scratch(size_for_shape(&[suffix_length, k], self.data_type))?;
+
+ if suffix_length > 0 && k > 0 {
+ encoder.encode_fill(&mut topk_ids, 0xFF);
+ }
+
if suffix_length > 0 && e > 0 && k > 0 {
- // Use the fused router+topk kernel
self.router_topk_kernel.encode(
- main_buf.borrow().deref(),
- self.router.weights_buf.borrow().deref(),
- self.router.biases_buf.borrow().deref(),
- topk_ids_buf.borrow_mut().deref_mut(),
- topk_probs_buf.borrow_mut().deref_mut(),
+ &input,
+ &self.router.weights,
+ &self.router.biases,
+ &mut topk_ids,
+ &mut topk_probs,
suffix_length as u32,
self.model_dim as u32,
e as u32,
@@ -288,23 +251,35 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
);
}
+ let mut offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;
+ let mut sumk = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;
+ let scatter_entries = num_blocks * num_tiles * 512;
+ let mut partials = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;
self.counts_offsets_kernel.encode(
- topk_ids_buf.borrow().deref(),
- offsets_buf.borrow_mut().deref_mut(),
- sumk_buf.borrow_mut().deref_mut(),
- partials_buf.borrow_mut().deref_mut(),
+ &topk_ids,
+ &mut offsets,
+ &mut sumk,
+ &mut partials,
suffix_length as u32,
e as u32,
k as u32,
encoder,
);
- let num_blocks = ((suffix_length + 255) / 256).max(1);
- let num_tiles = ((e + 512 - 1) / 512).max(1);
+ let mut block_bases = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;
+ let mut block_alloc = encoder.allocate_scratch(size_for_shape(&[num_blocks * num_tiles], DataType::U32))?;
+ let mut bucketed_ids = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;
+ let mut bucketed_probs = encoder.allocate_scratch(size_for_shape(&[total_rows], self.data_type))?;
+ let mut tok2row = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;
+
+ if suffix_length > 0 && k > 0 {
+ encoder.encode_fill(&mut tok2row, 0xFF);
+ }
+
self.scatter_bases_kernel.encode(
- partials_buf.borrow().deref(),
- block_bases_buf.borrow_mut().deref_mut(),
- block_alloc_buf.borrow_mut().deref_mut(),
+ &partials,
+ &mut block_bases,
+ &mut block_alloc,
e as u32,
num_blocks as u32,
num_tiles as u32,
@@ -312,30 +287,35 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
encoder,
);
self.scatter_map_kernel.encode(
- topk_ids_buf.borrow().deref(),
- topk_probs_buf.borrow().deref(),
- offsets_buf.borrow().deref(),
- block_bases_buf.borrow().deref(),
- block_alloc_buf.borrow().deref(),
- bucketed_ids_buf.borrow_mut().deref_mut(),
- bucketed_probs_buf.borrow_mut().deref_mut(),
+ &topk_ids,
+ &topk_probs,
+ &offsets,
+ &block_bases,
+ &block_alloc,
+ &mut bucketed_ids,
+ &mut bucketed_probs,
suffix_length as u32,
e as u32,
k as u32,
num_blocks as u32,
num_tiles as u32,
- tok2row_buf.borrow_mut().deref_mut(),
+ &mut tok2row,
encoder,
);
+ let mut x_perm = encoder.allocate_scratch(size_for_shape(&[total_rows, self.model_dim], self.data_type))?;
+ if suffix_length > 0 && k > 0 {
+ encoder.encode_fill(&mut x_perm, 0);
+ }
+
self.gather_kernels.encode(
encoder,
self.data_type,
MoeGatherArguments {
- x_buffer: main_buf.borrow().deref(),
- bucketed_ids_buffer: bucketed_ids_buf.borrow().deref(),
- x_perm_buffer: x_perm_buf.borrow_mut().deref_mut(),
- sumk_buffer: sumk_buf.borrow().deref(),
+ x: &input,
+ bucketed_ids: &bucketed_ids,
+ x_perm: &mut x_perm,
+ sumk: &sumk,
t: suffix_length,
k,
d_model: self.model_dim,
@@ -351,27 +331,33 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
let up_clip_max = self.moe_config.expert_config.up_clipping[1];
let silu_alpha = self.moe_config.expert_config.activation.alpha();
- if suffix_length == 1 {
- let total_rows = suffix_length * k;
+ let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::U32))?;
+ let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;
+ let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;
+ let h_blocks = self.hidden_dim.div_ceil(4);
+ let mut tile_map =
+ encoder.allocate_scratch(size_for_shape(&[total_rows * h_blocks.max(1) * 3], DataType::U32))?;
+ let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;
+ let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;
+
+ let y_partial = if suffix_length == 1 {
let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
self.experts_two_pass_decode_kernel.encode(
encoder,
MoeExpertsTwoPassArguments {
- x_perm_buffer: x_perm_buf.borrow().deref(),
- expert_offsets: offsets_buf.borrow().deref(),
- row_expert_map: row_expert_map_buf.borrow_mut().deref_mut(),
- hidden_buffer: hidden_buf.borrow_mut().deref_mut(),
- output_buffer: y_partial_buf.borrow_mut().deref_mut(),
- w13_all: self.shared_weights.w13_buf.borrow().deref(),
- w2_all: self.shared_weights.w2_buf.borrow().deref(),
- up_biases: self.shared_weights.up_biases_buf.borrow().deref(),
- down_biases: self.shared_weights.down_biases_buf.borrow().deref(),
- tile_counts: tile_counts_buf.borrow_mut().deref_mut(),
- tile_offsets: tile_offsets_buf.borrow_mut().deref_mut(),
- tile_map: tile_map_buf.borrow_mut().deref_mut(),
- total_tiles: total_tiles_buf.borrow_mut().deref_mut(),
- dispatch_args: dispatch_args_buf.borrow_mut().deref_mut(),
+ x_perm: &x_perm,
+ expert_offsets: &offsets,
+ row_expert_map: &mut row_expert_map,
+ w13_all: &self.shared_weights.w13,
+ w2_all: &self.shared_weights.w2,
+ up_biases: &self.shared_weights.up_biases,
+ down_biases: &self.shared_weights.down_biases,
+ tile_counts: &mut tile_counts,
+ tile_offsets: &mut tile_offsets,
+ tile_map: &mut tile_map,
+ total_tiles: &mut total_tiles,
+ dispatch_args: &mut dispatch_args,
total_rows,
d_model: self.model_dim,
d_ff: self.hidden_dim,
@@ -385,41 +371,22 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
silu_alpha,
data_type: self.data_type,
},
- );
+ )?
} else {
- let total_rows = suffix_length * k;
let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
-
- let x_perm_borrow = x_perm_buf.borrow();
- let offsets_borrow = offsets_buf.borrow();
- let mut row_expert_map_borrow = row_expert_map_buf.borrow_mut();
- let mut hidden_borrow = hidden_buf.borrow_mut();
- let mut y_partial_borrow = y_partial_buf.borrow_mut();
- let w13_borrow = self.shared_weights.w13_buf.borrow();
- let w2_borrow = self.shared_weights.w2_buf.borrow();
- let up_biases_borrow = self.shared_weights.up_biases_buf.borrow();
- let down_biases_borrow = self.shared_weights.down_biases_buf.borrow();
- let mut tile_counts_borrow = tile_counts_buf.borrow_mut();
- let mut tile_offsets_borrow = tile_offsets_buf.borrow_mut();
- let mut tile_map_borrow = tile_map_buf.borrow_mut();
- let mut total_tiles_borrow = total_tiles_buf.borrow_mut();
- let mut dispatch_args_borrow = dispatch_args_buf.borrow_mut();
-
let args = MoeExpertsTwoPassArguments {
- x_perm_buffer: x_perm_borrow.deref(),
- expert_offsets: offsets_borrow.deref(),
- row_expert_map: row_expert_map_borrow.deref_mut(),
- hidden_buffer: hidden_borrow.deref_mut(),
- output_buffer: y_partial_borrow.deref_mut(),
- w13_all: w13_borrow.deref(),
- w2_all: w2_borrow.deref(),
- up_biases: up_biases_borrow.deref(),
- down_biases: down_biases_borrow.deref(),
- tile_counts: tile_counts_borrow.deref_mut(),
- tile_offsets: tile_offsets_borrow.deref_mut(),
- tile_map: tile_map_borrow.deref_mut(),
- total_tiles: total_tiles_borrow.deref_mut(),
- dispatch_args: dispatch_args_borrow.deref_mut(),
+ x_perm: &x_perm,
+ expert_offsets: &offsets,
+ row_expert_map: &mut row_expert_map,
+ w13_all: &self.shared_weights.w13,
+ w2_all: &self.shared_weights.w2,
+ up_biases: &self.shared_weights.up_biases,
+ down_biases: &self.shared_weights.down_biases,
+ tile_counts: &mut tile_counts,
+ tile_offsets: &mut tile_offsets,
+ tile_map: &mut tile_map,
+ total_tiles: &mut total_tiles,
+ dispatch_args: &mut dispatch_args,
total_rows,
d_model: self.model_dim,
d_ff: self.hidden_dim,
@@ -433,21 +400,22 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
silu_alpha,
data_type: self.data_type,
};
- self.experts_two_pass_prefill_kernel.encode(encoder, args);
- }
+ self.experts_two_pass_prefill_kernel.encode(encoder, args)?
+ };
+ let mut output = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.model_dim], self.data_type))?;
self.finalize_kernel.encode(
- tok2row_buf.borrow().deref(),
- topk_probs_buf.borrow().deref(),
- y_partial_buf.borrow().deref(),
- main_buf.borrow_mut().deref_mut(),
+ &tok2row,
+ &topk_probs,
+ &y_partial,
+ &mut output,
suffix_length as u32,
self.model_dim as u32,
k as u32,
encoder,
);
- Ok(())
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rs
index 84f5b1b2d..18fd58478 100644
--- a/crates/backend-uzu/src/encodable_block/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mod.rs
@@ -1,4 +1,3 @@
-mod activation;
mod attention;
mod classifier_layer;
mod decoder;
@@ -17,19 +16,16 @@ mod rms_norm;
mod rope;
mod sampling;
mod short_conv_mixer;
-mod tensor_add_swap;
-mod tensor_copy;
-pub use activation::Activation;
-pub use attention::Attention;
+pub use attention::{Attention, AttentionArguments};
pub use classifier_layer::ClassifierLayer;
-pub use decoder::Decoder;
-pub(crate) use delta_net_mixer::DeltaNetMixer;
+pub use decoder::{Decoder, DecoderArguments, DecoderDecodeInput};
+pub(crate) use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer};
pub use embedding::Embedding;
-pub use layer::LayerExecutables;
+pub use layer::{LayerArguments, LayerExecutables};
pub use layer_norm::{LayerNorm, LayerNormError};
pub use linear::Linear;
-pub(crate) use mamba_mixer::MambaMixer;
+pub(crate) use mamba_mixer::{MambaArguments, MambaMixer};
pub use mlp::Mlp;
pub use normalization::Normalization;
pub use pooling::Pooling;
@@ -37,10 +33,10 @@ pub use prediction_head::ClassifierPredictionHead;
pub use qk_norm::QKNorm;
pub use rms_norm::{RMSNorm, RMSNormError};
pub use rope::Rope;
-pub use sampling::Sampling;
+pub(crate) use sampling::SamplingInputs;
+pub use sampling::{Sampling, SamplingArguments};
+pub(crate) use short_conv_mixer::ShortConvArguments;
pub use short_conv_mixer::ShortConvMixer;
-pub use tensor_add_swap::TensorAddSwap;
-pub use tensor_copy::TensorCopy;
#[derive(Clone)]
pub struct EncodingParameters {
@@ -53,12 +49,4 @@ impl EncodingParameters {
projection_step: None,
}
}
-
- pub fn with_projection(
- mut self,
- projection_step: usize,
- ) -> Self {
- self.projection_step = Some(projection_step);
- self
- }
}
diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rs
index d2349e401..ecc943705 100644
--- a/crates/backend-uzu/src/encodable_block/normalization.rs
+++ b/crates/backend-uzu/src/encodable_block/normalization.rs
@@ -5,9 +5,8 @@ use thiserror::Error;
use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};
use crate::{
DataType,
- backends::common::{Backend, Encoder},
+ backends::common::{Allocation, Backend, Encoder},
config::NormalizationConfig,
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
};
@@ -32,29 +31,18 @@ impl<B: Backend> Normalization<B> {
context: &B::Context,
intermediate_data_type: DataType,
config: NormalizationConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
) -> Result<Self, NormalizationError<B>> {
if config.subtract_mean {
- Ok(Self::LayerNorm(LayerNorm::new(
- context,
- intermediate_data_type,
- config,
- input_array_id,
- output_array_id,
- parameter_tree,
- )?))
+ Ok(Self::LayerNorm(LayerNorm::new(context, intermediate_data_type, config, parameter_tree)?))
} else {
Ok(Self::RMSNorm(RMSNorm::new(
context,
intermediate_data_type,
config,
- input_array_id,
- output_array_id,
parameter_tree,
None,
- None,
+ false,
false,
)?))
}
@@ -62,12 +50,14 @@ impl<B: Backend> Normalization<B> {
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: &Allocation<B>,
+ row_offset: usize,
+ row_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
+ ) -> Result<Allocation<B>, B::Error> {
match self {
- Self::LayerNorm(layer_norm) => layer_norm.encode(state, encoder),
- Self::RMSNorm(rms_norm) => rms_norm.encode(state, encoder),
+ Self::LayerNorm(layer_norm) => layer_norm.encode(input, row_offset, row_count, encoder),
+ Self::RMSNorm(rms_norm) => rms_norm.encode(input, row_offset, row_count, None, encoder),
}
}
}
diff --git a/crates/backend-uzu/src/encodable_block/pooling.rs b/crates/backend-uzu/src/encodable_block/pooling.rs
index 82ee17671..ced65d036 100644
--- a/crates/backend-uzu/src/encodable_block/pooling.rs
+++ b/crates/backend-uzu/src/encodable_block/pooling.rs
@@ -1,15 +1,13 @@
//! Pooling encodable for sequence-level aggregation.
-use std::ops::{Deref, DerefMut};
-
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{Kernels, PoolingClsKernel, PoolingMeanKernel},
},
config::PoolingType,
- forward_pass::state::{ArrayId, ForwardPassState},
};
enum PoolingKernel<B: Backend> {
@@ -20,8 +18,8 @@ enum PoolingKernel<B: Backend> {
impl<B: Backend> PoolingKernel<B> {
fn encode(
&self,
- input: &B::DenseBuffer,
- output: &mut B::DenseBuffer,
+ input: &Allocation<B>,
+ output: &mut Allocation<B>,
seq_len: u32,
hidden_dim: u32,
batch_dim: u32,
@@ -37,6 +35,7 @@ impl<B: Backend> PoolingKernel<B> {
pub struct Pooling<B: Backend> {
pooling_kernel: PoolingKernel<B>,
model_dim: usize,
+ data_type: DataType,
}
impl<B: Backend> Pooling<B> {
@@ -55,36 +54,20 @@ impl<B: Backend> Pooling<B> {
Ok(Self {
pooling_kernel,
model_dim,
+ data_type,
})
}
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ seq_len: usize,
+ main: &Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
+ ) -> Result<Allocation<B>, B::Error> {
let batch_dim = 1;
- let seq_len = state.aux_buffers_suffix_length();
-
- let main_array = state.array(ArrayId::Main);
- let pooling_array = state.array(ArrayId::ClassifierPooling);
-
- self.pooling_kernel.encode(
- main_array.buffer().borrow().deref(),
- pooling_array.buffer().borrow_mut().deref_mut(),
- seq_len as u32,
- self.model_dim as u32,
- batch_dim,
- encoder,
- );
-
- #[cfg(feature = "tracing")]
- {
- let output_pooling_trace = state.traces().borrow().output_pooling().clone();
- state.encode_copy_array(encoder, ArrayId::ClassifierPooling, output_pooling_trace);
- }
+ let mut pooling = encoder.allocate_scratch(size_for_shape(&[1, self.model_dim], self.data_type))?;
- state.set_active_row_count(pooling_array.shape()[0]);
- Ok(())
+ self.pooling_kernel.encode(main, &mut pooling, seq_len as u32, self.model_dim as u32, batch_dim, encoder);
+ Ok(pooling)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/prediction_head.rs b/crates/backend-uzu/src/encodable_block/prediction_head.rs
index aca2827c7..c918b2c3f 100644
--- a/crates/backend-uzu/src/encodable_block/prediction_head.rs
+++ b/crates/backend-uzu/src/encodable_block/prediction_head.rs
@@ -1,55 +1,67 @@
//! Prediction head encodable for classification output.
-#[cfg(feature = "tracing")]
-use crate::forward_pass::state::ArrayId;
use crate::{
- backends::common::{Backend, Encoder},
- encodable_block::{Activation, Normalization, linear::Linear},
- forward_pass::state::ForwardPassState,
+ DataType,
+ array::size_for_shape,
+ backends::common::{
+ ActivationConfig, Allocation, Backend, Encoder,
+ gpu_types::ActivationType,
+ kernel::{ActivationKernel, Kernels},
+ },
+ encodable_block::{Normalization, linear::Linear},
};
pub struct ClassifierPredictionHead<B: Backend> {
dense: Box<dyn Linear<B>>,
- activation: Activation<B>,
+ activation_kernel: <B::Kernels as Kernels>::ActivationKernel,
+ activation: ActivationConfig,
+ activation_data_type: DataType,
norm: Normalization<B>,
readout: Box<dyn Linear<B>>,
- #[allow(dead_code)]
- num_labels: usize,
+ hidden_dim: usize,
}
impl<B: Backend> ClassifierPredictionHead<B> {
pub fn new(
+ context: &B::Context,
dense: Box<dyn Linear<B>>,
- activation: Activation<B>,
+ activation: ActivationConfig,
+ activation_data_type: DataType,
norm: Normalization<B>,
readout: Box<dyn Linear<B>>,
- num_labels: usize,
- ) -> Self {
- Self {
+ hidden_dim: usize,
+ ) -> Result<Self, B::Error> {
+ let activation_kernel = <B::Kernels as Kernels>::ActivationKernel::new(context, activation_data_type, false)?;
+ Ok(Self {
dense,
+ activation_kernel,
activation,
+ activation_data_type,
norm,
readout,
- num_labels,
- }
+ hidden_dim,
+ })
}
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- self.dense.encode(state, encoder)?;
- self.activation.encode(state, encoder)?;
- self.norm.encode(state, encoder)?;
- self.readout.encode(state, encoder)?;
-
- #[cfg(feature = "tracing")]
- {
- let traces = state.traces().clone();
- state.encode_copy_array(encoder, ArrayId::ClassifierPredictionHeadLogits, traces.borrow().logits.clone());
+ ) -> Result<Allocation<B>, B::Error> {
+ let batch_dim = 1;
+ let dense = self.dense.encode(input, batch_dim, encoder)?;
+ if self.activation.act_type() == ActivationType::IDENTITY {
+ panic!("Identity activation is not supported for kernel");
}
-
- Ok(())
+ let mut activated = encoder.allocate_scratch(size_for_shape(&[self.hidden_dim], self.activation_data_type))?;
+ self.activation_kernel.encode(
+ Some(&dense),
+ &mut activated,
+ self.hidden_dim as u32,
+ self.activation.act_type(),
+ encoder,
+ );
+ let normalized = self.norm.encode(&activated, 0, batch_dim, encoder)?;
+ self.readout.encode(normalized, batch_dim, encoder)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/qk_norm.rs b/crates/backend-uzu/src/encodable_block/qk_norm.rs
index cbda452d7..a4c7b18c9 100644
--- a/crates/backend-uzu/src/encodable_block/qk_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/qk_norm.rs
@@ -1,21 +1,14 @@
//! QK Normalization encodable.
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
use thiserror::Error;
use crate::{
DataType,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{Kernels, QKNormKernel},
},
config::{NormalizationConfig, UpcastMode},
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -32,9 +25,8 @@ pub struct QKNorm<B: Backend> {
key_kernel: Option<<B::Kernels as Kernels>::QKNormKernel>,
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
- qkv_array_id: ArrayId,
- query_scales_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,
- key_scales_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,
+ query_scales: Option<Allocation<B>>,
+ key_scales: Option<Allocation<B>>,
num_q_heads: usize,
num_kv_heads: usize,
head_dim: usize,
@@ -46,7 +38,6 @@ impl<B: Backend> QKNorm<B> {
intermediate_data_type: DataType,
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
- qkv_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
num_q_heads: usize,
num_kv_heads: usize,
@@ -54,12 +45,16 @@ impl<B: Backend> QKNorm<B> {
) -> Result<Self, QKNormError<B>> {
let mut query_kernel = None;
let mut key_kernel = None;
- let mut query_scales_buffer = None;
- let mut key_scales_buffer = None;
+ let mut query_scales = None;
+ let mut key_scales = None;
// Setup query normalization if configured
if let Some(ref q_config) = query_config {
- let scales = parameter_tree.leaf_array("query_norm.scales").map_err(QKNormError::ParameterError)?;
+ let scales = parameter_tree
+ .leaf("query_norm.scales")
+ .map_err(QKNormError::ParameterError)?
+ .read_allocation()
+ .map_err(QKNormError::ParameterError)?;
let accumulation_data_type: DataType = q_config.accumulation_precision.into();
let scale_data_type: DataType = q_config.scale_precision.into();
@@ -79,12 +74,16 @@ impl<B: Backend> QKNorm<B> {
.map_err(QKNormError::BackendError)?;
query_kernel = Some(kernel);
- query_scales_buffer = Some(scales.buffer());
+ query_scales = Some(scales);
}
// Setup key normalization if configured
if let Some(ref k_config) = key_config {
- let scales = parameter_tree.leaf_array("key_norm.scales").map_err(QKNormError::ParameterError)?;
+ let scales = parameter_tree
+ .leaf("key_norm.scales")
+ .map_err(QKNormError::ParameterError)?
+ .read_allocation()
+ .map_err(QKNormError::ParameterError)?;
let accumulation_data_type: DataType = k_config.accumulation_precision.into();
let scale_data_type: DataType = k_config.scale_precision.into();
@@ -104,7 +103,7 @@ impl<B: Backend> QKNorm<B> {
.map_err(QKNormError::BackendError)?;
key_kernel = Some(kernel);
- key_scales_buffer = Some(scales.buffer());
+ key_scales = Some(scales);
}
Ok(Self {
@@ -112,9 +111,8 @@ impl<B: Backend> QKNorm<B> {
key_kernel,
query_config,
key_config,
- qkv_array_id,
- query_scales_buffer,
- key_scales_buffer,
+ query_scales,
+ key_scales,
num_q_heads,
num_kv_heads,
head_dim,
@@ -123,21 +121,19 @@ impl<B: Backend> QKNorm<B> {
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ qkv: &mut Allocation<B>,
+ batch_dim: usize,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let qkv_array = state.array(self.qkv_array_id);
- let batch_dim = qkv_array.shape()[0] as u32;
-
// Process query normalization if configured
- if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =
- (&self.query_kernel, &self.query_scales_buffer, &self.query_config)
+ if let (Some(query_kernel), Some(query_scales), Some(query_config)) =
+ (&self.query_kernel, &self.query_scales, &self.query_config)
{
query_kernel.encode(
- None::<&B::DenseBuffer>,
- query_scales_buffer.borrow().deref(),
- qkv_array.buffer().borrow_mut().deref_mut(),
- batch_dim,
+ None::<&Allocation<B>>,
+ query_scales,
+ &mut *qkv,
+ batch_dim as u32,
self.num_q_heads as u32,
self.num_kv_heads as u32,
self.head_dim as u32,
@@ -151,14 +147,14 @@ impl<B: Backend> QKNorm<B> {
}
// Process key normalization if configured
- if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =
- (&self.key_kernel, &self.key_scales_buffer, &self.key_config)
+ if let (Some(key_kernel), Some(key_scales), Some(key_config)) =
+ (&self.key_kernel, &self.key_scales, &self.key_config)
{
key_kernel.encode(
- None::<&B::DenseBuffer>,
- key_scales_buffer.borrow().deref(),
- qkv_array.buffer().borrow_mut().deref_mut(),
- batch_dim,
+ None::<&Allocation<B>>,
+ key_scales,
+ &mut *qkv,
+ batch_dim as u32,
self.num_q_heads as u32,
self.num_kv_heads as u32,
self.head_dim as u32,
diff --git a/crates/backend-uzu/src/encodable_block/rms_norm.rs b/crates/backend-uzu/src/encodable_block/rms_norm.rs
index 4d07a7242..a45d4fafb 100644
--- a/crates/backend-uzu/src/encodable_block/rms_norm.rs
+++ b/crates/backend-uzu/src/encodable_block/rms_norm.rs
@@ -1,21 +1,15 @@
//! RMS Normalization encodable.
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
use thiserror::Error;
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{Kernels, RMSNormKernel},
},
config::{NormalizationConfig, UpcastMode},
- forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -30,12 +24,11 @@ pub enum RMSNormError<B: Backend> {
pub struct RMSNorm<B: Backend> {
kernel: <B::Kernels as Kernels>::RMSNormKernel,
config: NormalizationConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- shortcut_array_id: Option<ArrayId>,
- scales_buffer: Rc<RefCell<B::DenseBuffer>>,
- hadamard_factors_buffer: Option<B::DenseBuffer>,
- use_sampling_range: bool,
+ scales: Allocation<B>,
+ element_count: usize,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ hadamard_factors: Option<Allocation<B>>,
}
impl<B: Backend> RMSNorm<B> {
@@ -43,14 +36,14 @@ impl<B: Backend> RMSNorm<B> {
context: &B::Context,
intermediate_data_type: DataType,
config: NormalizationConfig,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
- hadamard_factors_buffer: Option<B::DenseBuffer>,
- shortcut_array_id: Option<ArrayId>,
+ hadamard_factors: Option<Allocation<B>>,
+ use_shortcut: bool,
residual_add: bool,
) -> Result<Self, RMSNormError<B>> {
- let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;
+ let scales_leaf = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;
+ let element_count = scales_leaf.shape()[0];
+ let scales = scales_leaf.read_allocation().map_err(RMSNormError::ParameterError)?;
let accumulation_data_type: DataType = config.accumulation_precision.into();
let scale_data_type: DataType = config.scale_precision.into();
@@ -66,82 +59,50 @@ impl<B: Backend> RMSNorm<B> {
scales_type,
output_type,
accumulation_data_type,
- input_array_id == output_array_id,
+ false,
config.upcast_mode == UpcastMode::FullLayer,
- shortcut_array_id.is_some(),
+ use_shortcut,
residual_add,
- hadamard_factors_buffer.is_some(),
+ hadamard_factors.is_some(),
)
.map_err(RMSNormError::BackendError)?;
Ok(Self {
kernel,
config,
- input_array_id,
- output_array_id,
- shortcut_array_id,
- scales_buffer: scales.buffer(),
- hadamard_factors_buffer,
- use_sampling_range: false,
+ scales,
+ element_count,
+ input_data_type: input_type,
+ output_data_type: output_type,
+ hadamard_factors,
})
}
- /// When enabled, this RMSNorm only runs on `state.sampling_start()..+state.sampling_length()`.
- /// This is useful for the final output norm before readout/sampling in prefill.
- pub fn with_sampling_range(mut self) -> Self {
- self.use_sampling_range = true;
- self
- }
-
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ input: &Allocation<B>,
+ row_offset: usize,
+ row_count: usize,
+ shortcut: Option<&mut Allocation<B>>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let input_array = state.array(self.input_array_id);
- let output_array = state.array(self.output_array_id);
-
- let suffix_length = input_array.shape()[0];
- let element_count = input_array.shape()[1];
-
- let input_elem_size = input_array.data_type().size_in_bytes();
- let output_elem_size = output_array.data_type().size_in_bytes();
-
- let (batch_start, batch_len) = if self.use_sampling_range {
- (state.sampling_start(), state.sampling_length())
- } else {
- (0, state.active_row_count())
- };
-
- let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));
- if batch_len == 0 {
- return Ok(());
- }
-
- let row_size_in_bytes = element_count * input_elem_size;
- let input_offset = batch_start * row_size_in_bytes;
-
- let output_row_size_in_bytes = element_count * output_elem_size;
- let output_offset = batch_start * output_row_size_in_bytes;
-
- let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());
- let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());
-
- let shortcut_rc = self.shortcut_array_id.map(|id| state.array(id).buffer());
- let mut shortcut_borrow = shortcut_rc.as_ref().map(|rc| rc.borrow_mut());
-
+ ) -> Result<Allocation<B>, B::Error> {
+ let row_size = self.element_count * self.input_data_type.size_in_bytes();
+ let row_offset_bytes = row_offset * row_size;
+ let shortcut = shortcut.map(|shortcut| (shortcut, row_offset_bytes));
+ let mut output =
+ encoder.allocate_scratch(size_for_shape(&[row_count, self.element_count], self.output_data_type))?;
self.kernel.encode(
- input_buffer_borrow.as_deref().map(|b| (b, input_offset)),
- self.scales_buffer.borrow().deref(),
- (output_array.buffer().borrow_mut().deref_mut(), output_offset),
- shortcut_borrow.as_deref_mut().map(|b| (b, input_offset)),
- self.hadamard_factors_buffer.as_ref(),
- batch_len as u32,
- element_count as u32,
+ Some((input, row_offset_bytes)),
+ &self.scales,
+ &mut output,
+ shortcut,
+ self.hadamard_factors.as_ref(),
+ row_count as u32,
+ self.element_count as u32,
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
encoder,
);
- Ok(())
+ Ok(output)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rs
index 67978e3af..ed22dd8ca 100644
--- a/crates/backend-uzu/src/encodable_block/rope.rs
+++ b/crates/backend-uzu/src/encodable_block/rope.rs
@@ -1,19 +1,19 @@
//! Rope (Rotary Position Embedding) encodable.
-use std::ops::{Deref, DerefMut};
-
use crate::{
DataType,
+ array::size_for_shape,
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::{Kernels, RopeKernel},
},
- forward_pass::state::{ArrayId, ForwardPassState, RopeType},
+ forward_pass::state::RopeType,
};
pub struct Rope<B: Backend> {
kernel: <B::Kernels as Kernels>::RopeKernel,
rope_type: RopeType,
+ data_type: DataType,
}
impl<B: Backend> Rope<B> {
@@ -25,51 +25,54 @@ impl<B: Backend> Rope<B> {
Ok(Self {
kernel: <B::Kernels as Kernels>::RopeKernel::new(context, data_type)?,
rope_type,
+ data_type,
})
}
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ qkv: &Allocation<B>,
+ token_positions: &Allocation<B>,
+ cosines: &Allocation<B>,
+ sines: &Allocation<B>,
+ suffix_length: usize,
+ num_heads: usize,
+ num_groups: usize,
+ head_dim: usize,
+ rope_max_seq_len: usize,
+ rope_dim: usize,
use_rope: bool,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let token_positions = state.array(ArrayId::TokenPositions);
- let qkv = state.array(ArrayId::QKV);
- let cosines = state.array(ArrayId::RopeCosines(self.rope_type));
- let sines = state.array(ArrayId::RopeSines(self.rope_type));
- let rotated_queries = state.array(ArrayId::RotatedQueries);
- let rotated_keys = state.array(ArrayId::RotatedKeys);
-
- let suffix_length = qkv.shape()[0];
-
- let rope_max_seq_len = cosines.shape()[0];
- let rope_dim = if use_rope {
- cosines.shape()[1]
+ ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
+ let applied_rope_dim = if use_rope {
+ assert!(rope_dim > 0, "enabled RoPE dimension must be greater than zero");
+ rope_dim
} else {
0
};
-
- let num_heads = rotated_queries.shape()[0];
- let head_dim = rotated_queries.shape()[2];
-
- let num_groups = rotated_keys.shape()[0];
-
+ let mut rotated_queries =
+ encoder.allocate_scratch(size_for_shape(&[num_heads, suffix_length, head_dim], self.data_type))?;
+ let mut rotated_keys =
+ encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;
self.kernel.encode(
- qkv.buffer().borrow().deref(),
- cosines.buffer().borrow().deref(),
- sines.buffer().borrow().deref(),
- (token_positions.buffer().borrow().deref(), token_positions.offset()),
- rotated_queries.buffer().borrow_mut().deref_mut(),
- rotated_keys.buffer().borrow_mut().deref_mut(),
+ qkv,
+ cosines,
+ sines,
+ token_positions,
+ &mut rotated_queries,
+ &mut rotated_keys,
head_dim as u32,
- rope_dim as u32,
+ applied_rope_dim as u32,
num_heads as u32,
num_groups as u32,
suffix_length as u32,
rope_max_seq_len as u32,
encoder,
);
- Ok(())
+ Ok((rotated_queries, rotated_keys))
+ }
+
+ pub fn rope_type(&self) -> RopeType {
+ self.rope_type
}
}
diff --git a/crates/backend-uzu/src/encodable_block/sampling.rs b/crates/backend-uzu/src/encodable_block/sampling.rs
index 26fdd9173..47c2c30ca 100644
--- a/crates/backend-uzu/src/encodable_block/sampling.rs
+++ b/crates/backend-uzu/src/encodable_block/sampling.rs
@@ -1,28 +1,79 @@
//! Sampling kernel encodable.
-use std::ops::{Deref, DerefMut};
+use ndarray::ArrayView2;
use crate::{
DataType,
+ array::{Array, ArrayContextExt},
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder,
kernel::sampling::{SamplingError, SamplingKernel},
},
- forward_pass::state::{ArrayId, ForwardPassState},
+ session::parameter::SamplingMethod,
};
pub struct Sampling<B: Backend> {
kernel: SamplingKernel<B>,
}
+pub struct SamplingArguments<'a, B: Backend> {
+ pub logits: &'a mut Allocation<B>,
+ pub seeds: &'a Allocation<B>,
+ pub bitmask: Option<&'a Allocation<B>>,
+ pub output: &'a mut Allocation<B>,
+ pub sampling_method: SamplingMethod,
+ pub batch_size: usize,
+ pub vocab_size: usize,
+}
+
+pub(crate) struct SamplingInputs<B: Backend> {
+ pub seeds: Array<B>,
+ pub bitmask: Option<Array<B>>,
+}
+
+impl<B: Backend> SamplingInputs<B> {
+ pub(crate) fn bitmask_array_from_slice(
+ context: &B::Context,
+ row_count: usize,
+ token_bitmask: &[u32],
+ bitmask_row_len: usize,
+ ) -> Array<B> {
+ let mut array = context.create_array_zeros(&[row_count, bitmask_row_len], DataType::U32);
+ let source =
+ ArrayView2::from_shape((row_count, bitmask_row_len), token_bitmask).expect("Invalid token bitmask shape");
+ array.copy_from_view(source);
+ array
+ }
+
+ pub(crate) fn from_slices(
+ context: &B::Context,
+ token_seeds: &[u64],
+ token_bitmask: Option<&[u32]>,
+ bitmask_row_len: Option<usize>,
+ ) -> Self {
+ let seeds = context.create_array_from(&[token_seeds.len()], token_seeds);
+
+ let bitmask = match (token_bitmask, bitmask_row_len) {
+ (Some(token_bitmask), Some(bitmask_row_len)) => {
+ Some(Self::bitmask_array_from_slice(context, token_seeds.len(), token_bitmask, bitmask_row_len))
+ },
+ (None, None) => None,
+ _ => panic!("bitmask data and row length must either both exist or both be absent"),
+ };
+
+ Self {
+ seeds,
+ bitmask,
+ }
+ }
+}
+
impl<B: Backend> Sampling<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
- max_batch_size: usize,
- max_vocab_size: usize,
) -> Result<Self, SamplingError<B>> {
- let kernel = SamplingKernel::new(context, data_type, max_batch_size, max_vocab_size)?;
+ let kernel = SamplingKernel::new(context, data_type)?;
Ok(Self {
kernel,
})
@@ -30,43 +81,19 @@ impl<B: Backend> Sampling<B> {
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ args: SamplingArguments<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let batch_dim = state.sampling_length();
- let sampling_start = state.sampling_start();
- let sampling_method = state.sampling_method().unwrap();
-
- let logits = state.array(ArrayId::Logits);
- let seeds = state.array(ArrayId::TokenSeeds);
- let output = state.sampling_output().unwrap();
-
- let vocab_size = logits.shape()[1];
- let seeds_offset = seeds.offset() + sampling_start * std::mem::size_of::<u64>();
-
- let (bitmask_buffer, bitmask_offset) = state.token_bitmask().map_or((None, 0usize), |bitmask| {
- let bitmask_row_len = bitmask.shape()[1];
- let bitmask_offset = bitmask.offset() + sampling_start * bitmask_row_len * std::mem::size_of::<u32>();
- (Some(bitmask.buffer()), bitmask_offset)
- });
- let bitmask_borrow = bitmask_buffer.as_ref().map(|b| b.borrow());
-
- self.kernel
- .encode(
- logits.buffer().borrow_mut().deref_mut(),
- seeds.buffer().borrow().deref(),
- seeds_offset,
- bitmask_borrow.as_deref(),
- bitmask_offset,
- output.buffer().borrow_mut().deref_mut(),
- sampling_method,
- batch_dim,
- vocab_size,
- encoder,
- )
- .expect("Sampling encoding failed");
-
- Ok(())
+ ) -> Result<(), SamplingError<B>> {
+ self.kernel.encode(
+ args.logits,
+ args.seeds,
+ args.bitmask,
+ args.output,
+ args.sampling_method,
+ args.batch_size,
+ args.vocab_size,
+ encoder,
+ )
}
}
diff --git a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
index 7d5318999..e7eca1e62 100644
--- a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs
@@ -1,20 +1,31 @@
-use std::ops::{Deref, DerefMut};
+use thiserror::Error;
use crate::{
DataType,
- array::Array,
+ array::size_for_shape,
backends::common::{
- Backend, Context, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
},
config::{DecoderLayerType, ShortConvConfig},
- encodable_block::linear::Linear,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::{ParameterTree, resolve_subtree},
+ encodable_block::linear::{Linear, LinearBlockError},
+ forward_pass::short_conv_layer::ShortConvLayer,
+ parameters::{ParameterLoaderError, ParameterTree, try_resolve_subtree},
};
+#[derive(Debug, Error)]
+pub enum ShortConvMixerError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Unsupported configuration: {0}")]
+ UnsupportedConfiguration(String),
+ #[error("Linear error: {0}")]
+ LinearError(#[from] Box<LinearBlockError<B>>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+}
+
pub struct ShortConvMixer<B: Backend> {
- layer_index: usize,
config: ShortConvConfig,
model_dim: usize,
in_projection: Box<dyn Linear<B>>,
@@ -23,8 +34,17 @@ pub struct ShortConvMixer<B: Backend> {
short_conv_prefill: <B::Kernels as Kernels>::ShortConvPrefillKernel,
short_conv_decode: <B::Kernels as Kernels>::ShortConvDecodeKernel,
short_conv_trie: <B::Kernels as Kernels>::ShortConvTrieKernel,
- conv_weight: Array<B>,
- conv_bias: Option<Array<B>>,
+ conv_weight: Allocation<B>,
+ conv_bias: Option<Allocation<B>>,
+ data_type: DataType,
+}
+
+pub(crate) struct ShortConvArguments<'a, B: Backend> {
+ pub active_row_count: usize,
+ pub sampling_start: usize,
+ pub sampling_length: usize,
+ pub token_parents: &'a Allocation<B>,
+ pub layer: &'a mut ShortConvLayer<B>,
}
impl<B: Backend> ShortConvMixer<B> {
@@ -35,60 +55,61 @@ impl<B: Backend> ShortConvMixer<B> {
layer_index: usize,
model_dim: usize,
decoder_layer_loader: &ParameterTree<B::Context>,
- ) -> (Self, Option<B::DenseBuffer>) {
+ ) -> Result<(Self, Option<Allocation<B>>), ShortConvMixerError<B>> {
if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {
- panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);
+ return Err(ShortConvMixerError::UnsupportedConfiguration(format!(
+ "layer {layer_index} marked as non-ShortConv but ShortConv config provided"
+ )));
+ }
+ if short_conv_config.kernel_size < 2 {
+ return Err(ShortConvMixerError::UnsupportedConfiguration(format!(
+ "kernel_size must be >= 2, got {}",
+ short_conv_config.kernel_size
+ )));
}
- let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);
- let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);
+ let mixer_tree = try_resolve_subtree(decoder_layer_loader, &["mixer"])?;
+ let conv_tree = try_resolve_subtree(&mixer_tree, &["conv"])?;
let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();
let (in_projection, in_proj_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(
&short_conv_config.in_projection_config,
- false,
model_dim,
[model_dim * 3],
context,
- &resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),
- ArrayId::Main,
- ArrayId::SsmInProj,
+ &try_resolve_subtree(&mixer_tree, &["in_projection", "in_proj"])?,
)
- .expect("Failed to create in-projection kernel");
+ .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;
let out_projection = <dyn Linear<B>>::new(
&short_conv_config.out_projection_config,
- false,
model_dim,
[model_dim],
context,
- &resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),
- ArrayId::AttentionOutput,
- ArrayId::Main,
+ &try_resolve_subtree(&mixer_tree, &["out_projection", "out_proj"])?,
)
- .expect("Failed to create out-projection kernel");
+ .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;
- let conv_weight = conv_tree.leaf_array("weights").unwrap().clone();
+ let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;
let conv_bias = if short_conv_config.conv_config.has_biases {
- Some(conv_tree.leaf_array("biases").unwrap().clone())
+ Some(conv_tree.leaf("biases")?.read_allocation()?)
} else {
None
};
let has_bias = short_conv_config.conv_config.has_biases;
let short_conv_pack = <B::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)
- .expect("Failed to create short conv pack kernel");
+ .map_err(ShortConvMixerError::BackendError)?;
let short_conv_prefill = <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)
- .expect("Failed to create short conv prefill kernel");
+ .map_err(ShortConvMixerError::BackendError)?;
let short_conv_decode = <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias, true)
- .expect("Failed to create short conv decode kernel");
+ .map_err(ShortConvMixerError::BackendError)?;
let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)
- .expect("Failed to create short conv trie kernel");
+ .map_err(ShortConvMixerError::BackendError)?;
- (
+ Ok((
Self {
- layer_index,
config: short_conv_config,
model_dim,
in_projection,
@@ -99,169 +120,130 @@ impl<B: Backend> ShortConvMixer<B> {
short_conv_trie,
conv_weight,
conv_bias,
+ data_type,
},
in_proj_input_hadamard_factors,
- )
- }
-
- fn clear_suffix_state_valid_range(
- &self,
- state: &ForwardPassState<B>,
- ) {
- let Some(cache_layers) = state.cache_layers() else {
- return;
- };
- let cache = cache_layers.borrow();
- let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");
- layer.clear_suffix_state_valid_range();
- }
-
- fn set_suffix_state_valid_range(
- &self,
- state: &ForwardPassState<B>,
- start: usize,
- len: usize,
- ) {
- let Some(cache_layers) = state.cache_layers() else {
- return;
- };
- let cache = cache_layers.borrow();
- let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");
- layer.set_suffix_state_valid_range(start, len);
+ ))
}
fn run_conv(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut ShortConvLayer<B>,
+ token_parents: &Allocation<B>,
+ sampling_start: usize,
+ sampling_length: usize,
+ in_proj: &Allocation<B>,
encoder: &mut Encoder<B>,
active_row_count: usize,
- ) {
- self.clear_suffix_state_valid_range(state);
+ ) -> Result<Allocation<B>, B::Error> {
+ layer.clear_suffix_state_valid_range();
+ let mut out = encoder.allocate_scratch(size_for_shape(&[active_row_count, self.model_dim], self.data_type))?;
if active_row_count == 1 {
- self.run_decode_conv(state, encoder, 1);
- return;
+ self.run_decode_conv(layer, in_proj, &mut out, encoder, 1)?;
+ return Ok(out);
}
- let sampling_len = state.sampling_length();
- if sampling_len == 0 {
- self.run_prefill_conv(state, encoder, active_row_count);
- return;
+ if sampling_length == 0 {
+ self.run_prefill_conv(layer, in_proj, &mut out, encoder, active_row_count)?;
+ return Ok(out);
}
- let sampling_start = state.sampling_start();
- let trie_len = sampling_len;
+ let trie_len = sampling_length;
if trie_len <= 1 {
- self.run_prefill_conv(state, encoder, active_row_count);
- return;
+ self.run_prefill_conv(layer, in_proj, &mut out, encoder, active_row_count)?;
+ return Ok(out);
}
if sampling_start > 0 {
if sampling_start == 1 {
- self.run_decode_conv(state, encoder, 1);
+ self.run_decode_conv(layer, in_proj, &mut out, encoder, 1)?;
} else {
- self.run_prefill_conv(state, encoder, sampling_start);
+ self.run_prefill_conv(layer, in_proj, &mut out, encoder, sampling_start)?;
}
}
- self.run_trie_conv(state, encoder, sampling_start, trie_len);
- self.set_suffix_state_valid_range(state, sampling_start, trie_len);
+ self.run_trie_conv(layer, token_parents, in_proj, &mut out, encoder, sampling_start, trie_len)?;
+ layer.set_suffix_state_valid_range(sampling_start, trie_len);
+ Ok(out)
}
fn run_prefill_conv(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut ShortConvLayer<B>,
+ in_proj: &Allocation<B>,
+ out: &mut Allocation<B>,
encoder: &mut Encoder<B>,
suffix_length: usize,
- ) {
+ ) -> Result<(), B::Error> {
if self.model_dim == 0 || suffix_length == 0 {
- return;
+ return Ok(());
}
- let in_proj = state.array(ArrayId::SsmInProj);
- let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));
- let out = state.array(ArrayId::AttentionOutput);
-
- let bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
-
let kernel_size = self.config.kernel_size;
- let state_stride = kernel_size.saturating_sub(1);
+ let state_stride = kernel_size - 1;
- // Allocate temporary padded buffer
- let data_type: DataType = self.config.in_projection_config.activation_precision().into();
- let element_size = data_type.size_in_bytes();
let padded_rows = state_stride + suffix_length;
- let padded_size = padded_rows * self.model_dim * element_size;
- let mut padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");
+ let mut padded = encoder.allocate_scratch(size_for_shape(&[padded_rows, self.model_dim], self.data_type))?;
self.short_conv_pack.encode(
- conv_state.buffer().borrow().deref(),
- in_proj.buffer().borrow().deref(),
- &mut padded_buf,
+ (&layer.conv_state, 0),
+ in_proj,
+ &mut padded,
state_stride as u32,
suffix_length as u32,
self.model_dim as u32 * 3,
self.model_dim as u32,
encoder,
);
-
self.short_conv_prefill.encode(
- &padded_buf,
- in_proj.buffer().borrow().deref(),
- self.conv_weight.buffer().borrow().deref(),
- bias_buf_borrow.as_deref(),
- out.buffer().borrow_mut().deref_mut(),
- conv_state.buffer().borrow_mut().deref_mut(),
+ &padded,
+ in_proj,
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ out,
+ &mut layer.conv_state,
suffix_length as u32,
kernel_size as u32,
self.model_dim as u32 * 3,
state_stride as u32,
self.model_dim as u32,
encoder,
- )
+ );
+ Ok(())
}
fn run_trie_conv(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut ShortConvLayer<B>,
+ token_parents: &Allocation<B>,
+ in_proj: &Allocation<B>,
+ out: &mut Allocation<B>,
encoder: &mut Encoder<B>,
sampling_start: usize,
trie_len: usize,
- ) {
+ ) -> Result<(), B::Error> {
if self.model_dim == 0 || trie_len == 0 {
- return;
+ return Ok(());
}
- let in_proj = state.array(ArrayId::SsmInProj);
- let parents = state.array(ArrayId::TokenParents);
- let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));
- let suffix_state = state.array(ArrayId::ShortConvSuffixState(self.layer_index));
- let out = state.array(ArrayId::AttentionOutput);
-
- let data_type: DataType = self.config.in_projection_config.activation_precision().into();
- let elem_bytes = data_type.size_in_bytes();
+ let elem_bytes = DataType::from(self.config.in_projection_config.activation_precision()).size_in_bytes();
let kernel_size = self.config.kernel_size;
- let state_stride = kernel_size.saturating_sub(1);
+ let state_stride = kernel_size - 1;
let in_proj_stride = self.model_dim * 3;
- let in_proj_offset = in_proj.offset() + sampling_start * in_proj_stride * elem_bytes;
- let out_offset = out.offset() + sampling_start * self.model_dim * elem_bytes;
- let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;
- let base_state_offset = conv_state.offset();
- let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();
- let trie_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
- let trie_bias_buf_borrow = trie_bias_buf_rc.as_ref().map(|rc| rc.borrow());
-
+ let in_proj_view = (in_proj, sampling_start * in_proj_stride * elem_bytes);
+ let parents_view = (token_parents, sampling_start * std::mem::size_of::<i32>());
+ let out_view = (&mut *out, sampling_start * self.model_dim * elem_bytes);
self.short_conv_trie.encode(
- (in_proj.buffer().borrow().deref(), in_proj_offset),
- self.conv_weight.buffer().borrow().deref(),
- trie_bias_buf_borrow.as_deref(),
- (conv_state.buffer().borrow().deref(), base_state_offset),
- (parents.buffer().borrow().deref(), parents_offset),
- (out.buffer().borrow_mut().deref_mut(), out_offset),
- (suffix_state.buffer().borrow_mut().deref_mut(), suffix_state_offset),
+ in_proj_view,
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ (&layer.conv_state, 0),
+ parents_view,
+ out_view,
+ (&mut layer.suffix_state, sampling_start * self.model_dim * state_stride * elem_bytes),
trie_len as u32,
kernel_size as u32,
in_proj_stride as u32,
@@ -269,60 +251,60 @@ impl<B: Backend> ShortConvMixer<B> {
self.model_dim as u32,
encoder,
);
+ Ok(())
}
fn run_decode_conv(
&self,
- state: &mut ForwardPassState<B>,
+ layer: &mut ShortConvLayer<B>,
+ in_proj: &Allocation<B>,
+ out: &mut Allocation<B>,
encoder: &mut Encoder<B>,
suffix_length: usize,
- ) {
+ ) -> Result<(), B::Error> {
if self.model_dim == 0 || suffix_length == 0 {
- return;
+ return Ok(());
}
- let in_proj = state.array(ArrayId::SsmInProj);
- let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));
- let out = state.array(ArrayId::AttentionOutput);
-
- let decode_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
- let decode_bias_buf_borrow = decode_bias_buf_rc.as_ref().map(|rc| rc.borrow());
-
let kernel_size = self.config.kernel_size;
- let state_stride = kernel_size.saturating_sub(1);
+ let state_stride = kernel_size - 1;
self.short_conv_decode.encode(
- in_proj.buffer().borrow().deref(),
- self.conv_weight.buffer().borrow().deref(),
- decode_bias_buf_borrow.as_deref(),
- None::<&B::DenseBuffer>,
- out.buffer().borrow_mut().deref_mut(),
- conv_state.buffer().borrow_mut().deref_mut(),
+ in_proj,
+ &self.conv_weight,
+ self.conv_bias.as_ref(),
+ None::<&Allocation<B>>,
+ out,
+ &mut layer.conv_state,
suffix_length as u32,
kernel_size as u32,
self.model_dim as u32 * 3,
state_stride as u32,
self.model_dim as u32,
encoder,
- )
+ );
+ Ok(())
}
pub fn encode(
&self,
- state: &mut ForwardPassState<B>,
+ args: ShortConvArguments<B>,
+ input: Allocation<B>,
encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let active_row_count = state.active_row_count();
- if active_row_count == 0 {
- return Ok(());
- }
-
- self.in_projection.encode(state, encoder)?;
-
- self.run_conv(state, encoder, active_row_count);
-
- self.out_projection.encode(state, encoder)?;
-
- Ok(())
+ ) -> Result<Allocation<B>, B::Error> {
+ let ShortConvArguments {
+ active_row_count,
+ sampling_start,
+ sampling_length,
+ token_parents,
+ layer,
+ } = args;
+ assert!(active_row_count > 0, "ShortConv mixer requires at least one active row");
+
+ let in_proj = self.in_projection.encode(input, active_row_count, encoder)?;
+ let conv_output =
+ self.run_conv(layer, token_parents, sampling_start, sampling_length, &in_proj, encoder, active_row_count)?;
+
+ self.out_projection.encode(conv_output, active_row_count, encoder)
}
}
diff --git a/crates/backend-uzu/src/encodable_block/tensor_add_swap.rs b/crates/backend-uzu/src/encodable_block/tensor_add_swap.rs
deleted file mode 100644
index 2320c2c4a..000000000
--- a/crates/backend-uzu/src/encodable_block/tensor_add_swap.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-//! Tensor add-swap encodable.
-
-use std::ops::DerefMut;
-
-use crate::{
- DataType,
- backends::common::{
- Backend, Encoder,
- kernel::{Kernels, TensorAddSwapKernel},
- },
- forward_pass::state::{ArrayId, ForwardPassState},
-};
-
-pub struct TensorAddSwap<B: Backend> {
- kernel: <B::Kernels as Kernels>::TensorAddSwapKernel,
- skip_array_id: ArrayId,
- main_array_id: ArrayId,
-}
-
-impl<B: Backend> TensorAddSwap<B> {
- pub fn new(
- context: &B::Context,
- data_type: DataType,
- skip_array_id: ArrayId,
- main_array_id: ArrayId,
- ) -> Result<Self, B::Error> {
- let kernel = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)?;
-
- Ok(Self {
- kernel,
- skip_array_id,
- main_array_id,
- })
- }
-
- pub fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let skip_array = state.array(self.skip_array_id);
- let main_array = state.array(self.main_array_id);
-
- let length = skip_array.num_elements();
-
- self.kernel.encode(
- skip_array.buffer().borrow_mut().deref_mut(),
- main_array.buffer().borrow_mut().deref_mut(),
- length as u32,
- encoder,
- );
- Ok(())
- }
-}
diff --git a/crates/backend-uzu/src/encodable_block/tensor_copy.rs b/crates/backend-uzu/src/encodable_block/tensor_copy.rs
deleted file mode 100644
index 69e6c9117..000000000
--- a/crates/backend-uzu/src/encodable_block/tensor_copy.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-//! Tensor copy encodable.
-
-use std::ops::{Deref, DerefMut};
-
-use crate::{
- DataType,
- backends::common::{
- Backend, Encoder,
- kernel::{Kernels, TensorCopyKernel},
- },
- forward_pass::state::{ArrayId, ForwardPassState},
-};
-
-pub struct TensorCopy<B: Backend> {
- kernel: <B::Kernels as Kernels>::TensorCopyKernel,
- source_array: ArrayId,
- destination_array: ArrayId,
-}
-
-impl<B: Backend> TensorCopy<B> {
- pub fn new(
- context: &B::Context,
- data_type: DataType,
- source_array: ArrayId,
- destination_array: ArrayId,
- ) -> Result<Self, B::Error> {
- let kernel = <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type)?;
-
- Ok(Self {
- kernel,
- source_array,
- destination_array,
- })
- }
-
- pub fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- encoder: &mut Encoder<B>,
- ) -> Result<(), B::Error> {
- let source_array = state.array(self.source_array);
- let destination_array = state.array(self.destination_array);
-
- let length = source_array.num_elements();
-
- self.kernel.encode(
- source_array.buffer().borrow().deref(),
- destination_array.buffer().borrow_mut().deref_mut(),
- length as u32,
- encoder,
- );
- Ok(())
- }
-}
diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rs
index f99d46aa8..26b06f4ab 100644
--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs
+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs
@@ -1,8 +1,8 @@
use std::cell::Cell;
use crate::{
- array::ArrayContextExt,
- backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
+ array::size_for_shape,
+ backends::common::{AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
config::DecoderLayerType,
forward_pass::{
delta_net_layer::DeltaNetLayer,
@@ -13,7 +13,6 @@ use crate::{
},
};
-#[derive(Debug)]
pub enum CacheLayer<B: Backend> {
Transformer(KVCacheLayer<B>),
StateSpace(SSMLayer<B>),
@@ -21,7 +20,6 @@ pub enum CacheLayer<B: Backend> {
DeltaNet(DeltaNetLayer<B>),
}
-#[derive(Clone)]
pub enum CacheLayerSlice<B: Backend> {
Transformer(KVSlice<B>),
StateSpace,
@@ -29,15 +27,6 @@ pub enum CacheLayerSlice<B: Backend> {
DeltaNet,
}
-const ARRAY_TRANSFORMER_KEYS_LABEL: &str = "cache_layers_transformer_keys";
-const ARRAY_TRANSFORMER_VALUES_LABEL: &str = "cache_layers_transformer_values";
-const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str = "cache_layers_state_space_conv_state";
-const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str = "cache_layers_state_space_ssm_state";
-const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str = "cache_layers_short_conv_conv_state";
-const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str = "cache_layers_short_conv_suffix_state";
-const ARRAY_DELTA_NET_CONV_STATE_LABEL: &str = "cache_layers_delta_net_conv_state";
-const ARRAY_DELTA_NET_SSM_STATE_LABEL: &str = "cache_layers_delta_net_ssm_state";
-
impl<B: Backend> CacheLayer<B> {
pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {
match self {
@@ -102,7 +91,6 @@ pub struct CacheLayers<B: Backend> {
pub data: Box<[CacheLayer<B>]>,
}
-#[derive(Clone)]
pub struct CacheLayersSlice<B: Backend> {
pub layers: Vec<CacheLayerSlice<B>>,
}
@@ -118,7 +106,7 @@ impl<B: Backend> CacheLayers<B> {
let kv_shapes: Vec<[usize; 3]> =
model_shape.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();
- let data: Box<[CacheLayer<B>]> = model_shape
+ let mut data: Box<[CacheLayer<B>]> = model_shape
.layer_types()
.iter()
.enumerate()
@@ -139,19 +127,18 @@ impl<B: Backend> CacheLayers<B> {
prefix_len: 0,
}
};
+ let kv_bytes = size_for_shape(&shape, model_shape.kv_cache_data_type());
CacheLayer::Transformer(KVCacheLayer {
state: state.clone(),
- keys: context.create_array_zeros(
- &shape,
- model_shape.kv_cache_data_type(),
- &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),
- ),
- values: context.create_array_zeros(
- &shape,
- model_shape.kv_cache_data_type(),
- &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),
- ),
+ keys: context
+ .create_allocation(kv_bytes, AllocationType::Global)
+ .expect("Failed to create kv keys allocation"),
+ values: context
+ .create_allocation(kv_bytes, AllocationType::Global)
+ .expect("Failed to create kv values allocation"),
+ shape,
+ data_type: model_shape.kv_cache_data_type(),
})
},
DecoderLayerType::StateSpace {
@@ -165,39 +152,43 @@ impl<B: Backend> CacheLayers<B> {
let conv_shape = [*conv_dim, kernel_size.saturating_sub(1)];
let ssm_shape = [*num_heads, *head_dim, *state_dim];
let dtype = model_shape.activation_data_type();
+ let conv_bytes = size_for_shape(&conv_shape, dtype);
+ let ssm_bytes = size_for_shape(&ssm_shape, dtype);
CacheLayer::StateSpace(SSMLayer {
- conv_state: context.create_array_zeros(
- &conv_shape,
- dtype,
- &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),
- ),
- ssm_state: context.create_array_zeros(
- &ssm_shape,
- dtype,
- &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),
- ),
+ conv_state: (conv_bytes > 0).then(|| {
+ context
+ .create_allocation(conv_bytes, AllocationType::Global)
+ .expect("Failed to create ssm conv allocation")
+ }),
+ conv_shape,
+ ssm_state: context
+ .create_allocation(ssm_bytes, AllocationType::Global)
+ .expect("Failed to create ssm state allocation"),
+ ssm_shape,
+ data_type: dtype,
})
},
DecoderLayerType::ShortConv {
kernel_size,
} => {
- let conv_shape = [model_shape.model_dim(), kernel_size.saturating_sub(1)];
- let suffix_state_shape =
- [max_suffix_length, model_shape.model_dim(), kernel_size.saturating_sub(1)];
+ assert!(*kernel_size >= 2, "ShortConv kernel_size must be >= 2, got {}", kernel_size);
+ let conv_shape = [model_shape.model_dim(), kernel_size - 1];
+ let suffix_state_shape = [max_suffix_length, model_shape.model_dim(), kernel_size - 1];
let dtype = model_shape.activation_data_type();
+ let conv_bytes = size_for_shape(&conv_shape, dtype);
+ let suffix_bytes = size_for_shape(&suffix_state_shape, dtype);
CacheLayer::ShortConv(ShortConvLayer {
- conv_state: context.create_array_zeros(
- &conv_shape,
- dtype,
- &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),
- ),
- suffix_state: context.create_array_zeros(
- &suffix_state_shape,
- dtype,
- &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),
- ),
+ conv_state: context
+ .create_allocation(conv_bytes, AllocationType::Global)
+ .expect("Failed to create short conv allocation"),
+ conv_shape,
+ suffix_state: context
+ .create_allocation(suffix_bytes, AllocationType::Global)
+ .expect("Failed to create short conv suffix allocation"),
+ suffix_shape: suffix_state_shape,
+ data_type: dtype,
suffix_state_valid_start: Cell::new(0),
suffix_state_valid_len: Cell::new(0),
})
@@ -213,23 +204,49 @@ impl<B: Backend> CacheLayers<B> {
let conv_shape = [*conv_dim, kernel_size.saturating_sub(1)];
let ssm_shape = [*num_heads, *value_head_dim, *head_dim];
let dtype = model_shape.activation_data_type();
+ let conv_bytes = size_for_shape(&conv_shape, dtype);
+ let ssm_bytes = size_for_shape(&ssm_shape, dtype);
CacheLayer::DeltaNet(DeltaNetLayer {
- conv_state: context.create_array_zeros(
- &conv_shape,
- dtype,
- &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),
- ),
- ssm_state: context.create_array_zeros(
- &ssm_shape,
- dtype,
- &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),
- ),
+ conv_state: context
+ .create_allocation(conv_bytes, AllocationType::Global)
+ .expect("Failed to create delta net conv allocation"),
+ conv_shape,
+ ssm_state: context
+ .create_allocation(ssm_bytes, AllocationType::Global)
+ .expect("Failed to create delta net ssm allocation"),
+ ssm_shape,
+ data_type: dtype,
})
},
})
.collect();
+ let mut encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache initialization encoder");
+ for layer in data.iter_mut() {
+ match layer {
+ CacheLayer::Transformer(layer) => {
+ encoder.encode_fill(&mut layer.keys, 0);
+ encoder.encode_fill(&mut layer.values, 0);
+ },
+ CacheLayer::StateSpace(layer) => {
+ if let Some(conv_state) = layer.conv_state.as_mut() {
+ encoder.encode_fill(conv_state, 0);
+ }
+ encoder.encode_fill(&mut layer.ssm_state, 0);
+ },
+ CacheLayer::ShortConv(layer) => {
+ encoder.encode_fill(&mut layer.conv_state, 0);
+ encoder.encode_fill(&mut layer.suffix_state, 0);
+ },
+ CacheLayer::DeltaNet(layer) => {
+ encoder.encode_fill(&mut layer.conv_state, 0);
+ encoder.encode_fill(&mut layer.ssm_state, 0);
+ },
+ }
+ }
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to initialize cache allocations");
+
Self {
max_suffix_length,
max_prefix_length,
@@ -237,7 +254,11 @@ impl<B: Backend> CacheLayers<B> {
}
}
- pub fn clear(&mut self) {
+ pub fn clear(
+ &mut self,
+ context: &B::Context,
+ ) {
+ let mut encoder: Option<Encoder<B>> = None;
for layer in self.data.iter_mut() {
match layer {
CacheLayer::Transformer(layer) => match &mut layer.state {
@@ -255,11 +276,32 @@ impl<B: Backend> CacheLayers<B> {
*ring_length = 0;
},
},
- CacheLayer::StateSpace(layer) => layer.zero(),
- CacheLayer::ShortConv(layer) => layer.zero(),
- CacheLayer::DeltaNet(layer) => layer.zero(),
+ CacheLayer::StateSpace(layer) => {
+ let encoder = encoder
+ .get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));
+ if let Some(conv_state) = layer.conv_state.as_mut() {
+ encoder.encode_fill(conv_state, 0);
+ }
+ encoder.encode_fill(&mut layer.ssm_state, 0);
+ },
+ CacheLayer::ShortConv(layer) => {
+ let encoder = encoder
+ .get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));
+ encoder.encode_fill(&mut layer.conv_state, 0);
+ encoder.encode_fill(&mut layer.suffix_state, 0);
+ layer.clear_suffix_state_valid_range();
+ },
+ CacheLayer::DeltaNet(layer) => {
+ let encoder = encoder
+ .get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));
+ encoder.encode_fill(&mut layer.conv_state, 0);
+ encoder.encode_fill(&mut layer.ssm_state, 0);
+ },
}
}
+ if let Some(encoder) = encoder {
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to clear cache layers");
+ }
}
pub fn max_suffix_length(&self) -> usize {
@@ -282,7 +324,7 @@ impl<B: Backend> CacheLayers<B> {
if let Some(layer) = layer.as_transformer_mut() {
layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);
} else if let Some(layer) = layer.as_short_conv_mut() {
- layer.commit_from_suffix_state_if_valid(short_conv_commit_index);
+ layer.commit_from_suffix_state_if_valid(short_conv_commit_index, encoder);
}
}
}
@@ -340,21 +382,94 @@ impl<B: Backend> CacheLayers<B> {
}
}
+ pub fn copy_from(
+ &mut self,
+ source: &Self,
+ context: &B::Context,
+ ) {
+ self.copy_metadata_and_windowed_data_from(source, context);
+
+ let mut encoder = Encoder::new(context).expect("Failed to create cache layer copy encoder");
+ self.encode_copy_data_from(source, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to copy cache layers");
+ }
+
+ fn copy_metadata_and_windowed_data_from(
+ &mut self,
+ source: &Self,
+ context: &B::Context,
+ ) {
+ assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");
+
+ for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {
+ match (source_layer, destination_layer) {
+ (CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {
+ let copy_rows = source.prefix_segment_length();
+ if copy_rows > 0 && matches!(source.state, KVCacheLayerState::Windowed { .. }) {
+ let slice = source.slice(context, 0..copy_rows).expect("Failed to slice KV cache layer");
+ destination.apply_slice(&slice, None);
+ }
+ destination.state = source.state.clone();
+ },
+ (CacheLayer::StateSpace(_), CacheLayer::StateSpace(_)) => {},
+ (CacheLayer::ShortConv(_), CacheLayer::ShortConv(destination)) => {
+ destination.clear_suffix_state_valid_range();
+ },
+ (CacheLayer::DeltaNet(_), CacheLayer::DeltaNet(_)) => {},
+ _ => panic!("cache layer type mismatch while copying cache layers"),
+ }
+ }
+ }
+
+ fn encode_copy_data_from(
+ &mut self,
+ source: &Self,
+ encoder: &mut Encoder<B>,
+ ) {
+ assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");
+
+ for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {
+ match (source_layer, destination_layer) {
+ (CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {
+ if matches!(source.state, KVCacheLayerState::Full { .. }) {
+ source.encode_copy_prefix_rows_to(destination, source.prefix_segment_length(), encoder);
+ }
+ },
+ (CacheLayer::StateSpace(source), CacheLayer::StateSpace(destination)) => {
+ match (source.conv_state.as_ref(), destination.conv_state.as_mut()) {
+ (Some(source_conv_state), Some(destination_conv_state)) => {
+ encoder.encode_copy(source_conv_state, .., destination_conv_state, ..);
+ },
+ (None, None) => {},
+ _ => panic!("state-space conv_state presence mismatch while copying cache layers"),
+ }
+ encoder.encode_copy(&source.ssm_state, .., &mut destination.ssm_state, ..);
+ },
+ (CacheLayer::ShortConv(source), CacheLayer::ShortConv(destination)) => {
+ encoder.encode_copy(&source.conv_state, .., &mut destination.conv_state, ..);
+ },
+ (CacheLayer::DeltaNet(source), CacheLayer::DeltaNet(destination)) => {
+ encoder.encode_copy(&source.conv_state, .., &mut destination.conv_state, ..);
+ encoder.encode_copy(&source.ssm_state, .., &mut destination.ssm_state, ..);
+ },
+ _ => panic!("cache layer type mismatch while copying cache layers"),
+ }
+ }
+ }
+
pub fn clone(
&self,
context: &B::Context,
) -> Self {
let mut max_prefix_capacity_across_layers = 0usize;
- let data: Box<[CacheLayer<B>]> = self
+ let mut data: Box<[CacheLayer<B>]> = self
.data
.iter()
- .enumerate()
- .map(|(layer_index, layer)| match layer {
+ .map(|layer| match layer {
CacheLayer::Transformer(layer) => {
- let shape = layer.keys.shape().to_vec();
- let num_groups = shape[1];
- let head_dim = shape[2];
- let dtype = layer.keys.data_type();
+ let shape = layer.shape;
+ let [_, num_groups, head_dim] = shape;
+ let dtype = layer.data_type;
let copy_rows = layer.prefix_segment_length();
let new_total_len = copy_rows + self.max_suffix_length;
@@ -363,108 +478,104 @@ impl<B: Backend> CacheLayers<B> {
}
let new_shape = [new_total_len, num_groups, head_dim];
- let mut new_keys = context.create_array_zeros(
- &new_shape,
- dtype,
- &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),
- );
- let mut new_values = context.create_array_zeros(
- &new_shape,
- dtype,
- &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),
- );
-
- if copy_rows > 0 {
- new_keys.copy_slice(&layer.keys, 0, 0..copy_rows, 0);
- new_values.copy_slice(&layer.values, 0, 0..copy_rows, 0);
- }
+ let new_bytes = size_for_shape(&new_shape, dtype);
+ let new_keys = context
+ .create_allocation(new_bytes, AllocationType::Global)
+ .expect("Failed to create kv keys clone allocation");
+ let new_values = context
+ .create_allocation(new_bytes, AllocationType::Global)
+ .expect("Failed to create kv values clone allocation");
CacheLayer::Transformer(KVCacheLayer {
state: layer.state.clone(),
keys: new_keys,
values: new_values,
+ shape: new_shape,
+ data_type: dtype,
})
},
CacheLayer::StateSpace(layer) => {
- let conv_shape = layer.conv_state.shape().to_vec();
- let conv_dtype = layer.conv_state.data_type();
- let mut new_conv = context.create_array_uninitialized(
- &conv_shape,
- conv_dtype,
- &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),
- );
- new_conv.copy_from_array(&layer.conv_state);
-
- let ssm_shape = layer.ssm_state.shape().to_vec();
- let ssm_dtype = layer.ssm_state.data_type();
- let mut new_ssm = context.create_array_uninitialized(
- &ssm_shape,
- ssm_dtype,
- &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),
- );
- new_ssm.copy_from_array(&layer.ssm_state);
+ let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);
+ let ssm_bytes = size_for_shape(&layer.ssm_shape, layer.data_type);
+ let new_conv = (conv_bytes > 0).then(|| {
+ context
+ .create_allocation(conv_bytes, AllocationType::Global)
+ .expect("Failed to create ssm conv clone allocation")
+ });
+ let new_ssm = context
+ .create_allocation(ssm_bytes, AllocationType::Global)
+ .expect("Failed to create ssm state clone allocation");
CacheLayer::StateSpace(SSMLayer {
conv_state: new_conv,
+ conv_shape: layer.conv_shape,
ssm_state: new_ssm,
+ ssm_shape: layer.ssm_shape,
+ data_type: layer.data_type,
})
},
CacheLayer::ShortConv(layer) => {
- let conv_shape = layer.conv_state.shape().to_vec();
- let conv_dtype = layer.conv_state.data_type();
- let mut new_conv = context.create_array_uninitialized(
- &conv_shape,
- conv_dtype,
- &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),
- );
- new_conv.copy_from_array(&layer.conv_state);
-
- let suffix_shape = layer.suffix_state.shape().to_vec();
- let suffix_dtype = layer.suffix_state.data_type();
- let new_suffix = context.create_array_zeros(
- &suffix_shape,
- suffix_dtype,
- &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),
- );
+ let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);
+ let suffix_bytes = size_for_shape(&layer.suffix_shape, layer.data_type);
+ let new_conv = context
+ .create_allocation(conv_bytes, AllocationType::Global)
+ .expect("Failed to create short conv clone allocation");
+ let new_suffix = context
+ .create_allocation(suffix_bytes, AllocationType::Global)
+ .expect("Failed to create short conv suffix clone allocation");
CacheLayer::ShortConv(ShortConvLayer {
conv_state: new_conv,
+ conv_shape: layer.conv_shape,
suffix_state: new_suffix,
+ suffix_shape: layer.suffix_shape,
+ data_type: layer.data_type,
suffix_state_valid_start: Cell::new(0),
suffix_state_valid_len: Cell::new(0),
})
},
CacheLayer::DeltaNet(layer) => {
- let conv_shape = layer.conv_state.shape().to_vec();
- let conv_dtype = layer.conv_state.data_type();
- let mut new_conv = context.create_array_uninitialized(
- &conv_shape,
- conv_dtype,
- &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),
- );
- new_conv.copy_from_array(&layer.conv_state);
-
- let ssm_shape = layer.ssm_state.shape().to_vec();
- let ssm_dtype = layer.ssm_state.data_type();
- let mut new_ssm = context.create_array_uninitialized(
- &ssm_shape,
- ssm_dtype,
- &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),
- );
- new_ssm.copy_from_array(&layer.ssm_state);
+ let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);
+ let ssm_bytes = size_for_shape(&layer.ssm_shape, layer.data_type);
+ let new_conv = context
+ .create_allocation(conv_bytes, AllocationType::Global)
+ .expect("Failed to create delta net conv clone allocation");
+ let new_ssm = context
+ .create_allocation(ssm_bytes, AllocationType::Global)
+ .expect("Failed to create delta net ssm clone allocation");
CacheLayer::DeltaNet(DeltaNetLayer {
conv_state: new_conv,
+ conv_shape: layer.conv_shape,
ssm_state: new_ssm,
+ ssm_shape: layer.ssm_shape,
+ data_type: layer.data_type,
})
},
})
.collect();
- Self {
+ let mut zero_encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache clone zero encoder");
+ for layer in data.iter_mut() {
+ match layer {
+ CacheLayer::Transformer(layer) => {
+ zero_encoder.encode_fill(&mut layer.keys, 0);
+ zero_encoder.encode_fill(&mut layer.values, 0);
+ },
+ CacheLayer::ShortConv(layer) => {
+ zero_encoder.encode_fill(&mut layer.suffix_state, 0);
+ },
+ _ => {},
+ }
+ }
+ zero_encoder.end_encoding().submit().wait_until_completed().expect("Failed to zero cloned cache layers");
+
+ let mut cloned = Self {
max_suffix_length: self.max_suffix_length,
max_prefix_length: max_prefix_capacity_across_layers,
data,
- }
+ };
+ cloned.copy_from(self, context);
+ cloned
}
}
diff --git a/crates/backend-uzu/src/forward_pass/delta_net_layer.rs b/crates/backend-uzu/src/forward_pass/delta_net_layer.rs
index 4559e35cd..3fc31a82e 100644
--- a/crates/backend-uzu/src/forward_pass/delta_net_layer.rs
+++ b/crates/backend-uzu/src/forward_pass/delta_net_layer.rs
@@ -1,16 +1,12 @@
-use bytemuck::fill_zeroes;
+use crate::{
+ DataType,
+ backends::common::{Allocation, Backend},
+};
-use crate::{array::Array, backends::common::Backend};
-
-#[derive(Debug)]
pub struct DeltaNetLayer<B: Backend> {
- pub conv_state: Array<B>,
- pub ssm_state: Array<B>,
-}
-
-impl<B: Backend> DeltaNetLayer<B> {
- pub fn zero(&mut self) {
- fill_zeroes(self.conv_state.as_bytes_mut());
- fill_zeroes(self.ssm_state.as_bytes_mut());
- }
+ pub conv_state: Allocation<B>,
+ pub conv_shape: [usize; 2],
+ pub ssm_state: Allocation<B>,
+ pub ssm_shape: [usize; 3],
+ pub data_type: DataType,
}
diff --git a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
index dda8b3b80..a3dec827a 100644
--- a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
@@ -1,12 +1,12 @@
use crate::{
- array::{Array, ArrayContextExt},
+ DataType,
+ array::{Array, ArrayContextExt, size_for_shape},
backends::common::{
- Backend, Encoder,
+ Allocation, Backend, Encoder, allocation_as_bytes, allocation_as_bytes_mut,
kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},
},
};
-#[derive(Clone)]
pub enum KVSlice<B: Backend> {
Full {
base_prefix_len: usize,
@@ -16,8 +16,8 @@ pub enum KVSlice<B: Backend> {
base_ring_offset: usize,
base_ring_length: usize,
slots: Vec<usize>,
- keys: Array<B>, // [num_groups, slots.len(), head_dim]
- values: Array<B>, // [num_groups, slots.len(), head_dim]
+ keys: Array<B>, // [slots.len(), num_groups, head_dim]
+ values: Array<B>, // [slots.len(), num_groups, head_dim]
},
}
@@ -39,16 +39,60 @@ pub enum KVCacheLayerState {
pub const INVALID_POSITION: usize = i32::MAX as usize;
-#[derive(Debug)]
pub struct KVCacheLayer<B: Backend> {
pub state: KVCacheLayerState,
- /// [num_groups, max_prefix_length + max_suffix_length, head_dim]
- pub keys: Array<B>,
- /// [num_groups, max_prefix_length + max_suffix_length, head_dim]
- pub values: Array<B>,
+ /// [max_prefix_length + max_suffix_length, num_groups, head_dim]
+ pub keys: Allocation<B>,
+ /// [max_prefix_length + max_suffix_length, num_groups, head_dim]
+ pub values: Allocation<B>,
+ pub shape: [usize; 3],
+ pub data_type: DataType,
+}
+
+fn copy_rows(
+ source_keys_bytes: &[u8],
+ source_values_bytes: &[u8],
+ destination_keys_bytes: &mut [u8],
+ destination_values_bytes: &mut [u8],
+ row_size: usize,
+ row_pairs: impl IntoIterator<Item = (usize, usize)>,
+) {
+ for (source_row, destination_row) in row_pairs {
+ let source_offset = source_row * row_size;
+ let destination_offset = destination_row * row_size;
+ destination_keys_bytes[destination_offset..destination_offset + row_size]
+ .copy_from_slice(&source_keys_bytes[source_offset..source_offset + row_size]);
+ destination_values_bytes[destination_offset..destination_offset + row_size]
+ .copy_from_slice(&source_values_bytes[source_offset..source_offset + row_size]);
+ }
}
impl<B: Backend> KVCacheLayer<B> {
+ pub fn encode_copy_prefix_rows_to(
+ &self,
+ destination: &mut KVCacheLayer<B>,
+ row_count: usize,
+ encoder: &mut Encoder<B>,
+ ) {
+ if row_count == 0 {
+ return;
+ }
+
+ let [source_seq, num_groups, head_dim] = self.shape;
+ let [destination_seq, destination_groups, destination_head_dim] = destination.shape;
+ assert_eq!(num_groups, destination_groups, "KV cache group count mismatch");
+ assert_eq!(head_dim, destination_head_dim, "KV cache head dim mismatch");
+ assert_eq!(self.data_type, destination.data_type, "KV cache dtype mismatch");
+ assert!(row_count <= source_seq, "source KV cache copy exceeds source sequence");
+ assert!(row_count <= destination_seq, "source KV cache copy exceeds destination sequence");
+
+ let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);
+ let copy_size = row_count * row_size;
+
+ encoder.encode_copy(&self.keys, 0..copy_size, &mut destination.keys, 0..copy_size);
+ encoder.encode_copy(&self.values, 0..copy_size, &mut destination.values, 0..copy_size);
+ }
+
pub fn prefix_segment_length(&self) -> usize {
match &self.state {
KVCacheLayerState::Full {
@@ -139,7 +183,7 @@ impl<B: Backend> KVCacheLayer<B> {
}
fn scatter_if_required(
- &self,
+ &mut self,
source_indices: &[usize],
destination_indices: &[usize],
encoder: &mut Encoder<B>,
@@ -149,17 +193,15 @@ impl<B: Backend> KVCacheLayer<B> {
return;
}
- let k_shape = self.keys.shape();
- let v_shape = self.values.shape();
-
- let layer_data = KVLayerData {
- key_buffer: self.keys.buffer(),
- key_shape: [k_shape[0], k_shape[1], k_shape[2]],
- value_buffer: self.values.buffer(),
- value_shape: [v_shape[0], v_shape[1], v_shape[2]],
+ let mut layer_data = KVLayerData {
+ key_allocation: &mut self.keys,
+ key_shape: self.shape,
+ value_allocation: &mut self.values,
+ value_shape: self.shape,
};
- let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, encoder);
+ let _ =
+ kv_cache_update.encode(std::slice::from_mut(&mut layer_data), source_indices, destination_indices, encoder);
}
pub fn register_accepted_tokens(
@@ -204,16 +246,15 @@ impl<B: Backend> KVCacheLayer<B> {
if len == 0 || len > window_length {
return None;
}
- let shape = self.keys.shape();
- let num_groups = shape[1];
- let head_dim = shape[2];
- let dtype = self.keys.data_type();
+ let [_, num_groups, head_dim] = self.shape;
+ let dtype = self.data_type;
+ let row_size = size_for_shape(&[1, num_groups, head_dim], dtype);
let slice_shape = [len, num_groups, head_dim];
- let mut slice_keys =
- context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_keys");
- let mut slice_values =
- context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_values");
+ let mut slice_keys = context.create_array_uninitialized(&slice_shape, dtype);
+ let mut slice_values = context.create_array_uninitialized(&slice_shape, dtype);
+ let source_keys_bytes = allocation_as_bytes(&self.keys);
+ let source_values_bytes = allocation_as_bytes(&self.values);
let slots: Vec<usize> = (range.start..range.end)
.enumerate()
@@ -227,10 +268,17 @@ impl<B: Backend> KVCacheLayer<B> {
})
.collect();
- for (i, &slot) in slots.iter().enumerate() {
- slice_keys.copy_slice(&self.keys, 0, slot..slot + 1, i);
- slice_values.copy_slice(&self.values, 0, slot..slot + 1, i);
- }
+ let destination_keys_bytes = slice_keys.as_bytes_mut();
+ let destination_values_bytes = slice_values.as_bytes_mut();
+
+ copy_rows(
+ source_keys_bytes,
+ source_values_bytes,
+ destination_keys_bytes,
+ destination_values_bytes,
+ row_size,
+ slots.iter().enumerate().map(|(destination_row, &source_row)| (source_row, destination_row)),
+ );
Some(KVSlice::Window {
window_length,
@@ -281,15 +329,28 @@ impl<B: Backend> KVCacheLayer<B> {
},
) => {
*w_len = *window_length;
+ let [_, num_groups, head_dim] = self.shape;
+ let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);
+ let source_keys_bytes = keys.as_bytes();
+ let source_values_bytes = values.as_bytes();
+ let destination_keys_bytes = allocation_as_bytes_mut(&mut self.keys);
+ let destination_values_bytes = allocation_as_bytes_mut(&mut self.values);
match range {
None => {
*ring_offset = *base_ring_offset;
*ring_length = *base_ring_length;
- for (i, &slot) in slots.iter().enumerate() {
- self.keys.copy_slice(keys, 0, i..i + 1, slot);
- self.values.copy_slice(values, 0, i..i + 1, slot);
- }
+ copy_rows(
+ source_keys_bytes,
+ source_values_bytes,
+ destination_keys_bytes,
+ destination_values_bytes,
+ row_size,
+ slots
+ .iter()
+ .enumerate()
+ .map(|(source_row, &destination_row)| (source_row, destination_row)),
+ );
},
Some(r) => {
let len = r.end.saturating_sub(r.start);
@@ -316,11 +377,17 @@ impl<B: Backend> KVCacheLayer<B> {
*ring_offset = new_offset;
*ring_length = new_len;
- for (i, &slot) in slots[r.clone()].iter().enumerate() {
- let src_i = r.start + i;
- self.keys.copy_slice(keys, 0, src_i..src_i + 1, slot);
- self.values.copy_slice(values, 0, src_i..src_i + 1, slot);
- }
+ copy_rows(
+ source_keys_bytes,
+ source_values_bytes,
+ destination_keys_bytes,
+ destination_values_bytes,
+ row_size,
+ slots[r.clone()]
+ .iter()
+ .enumerate()
+ .map(|(index, &destination_row)| (r.start + index, destination_row)),
+ );
},
}
},
diff --git a/crates/backend-uzu/src/forward_pass/mod.rs b/crates/backend-uzu/src/forward_pass/mod.rs
index 474f782c6..3f599a5ec 100644
--- a/crates/backend-uzu/src/forward_pass/mod.rs
+++ b/crates/backend-uzu/src/forward_pass/mod.rs
@@ -5,8 +5,8 @@ pub mod delta_net_layer;
pub mod kv_cache_layer;
pub mod short_conv_layer;
pub mod ssm_layer;
+pub mod token_inputs;
-pub mod scratch_buffers;
pub mod state;
#[cfg(feature = "tracing")]
diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rs
index fc56220d3..d9a47b659 100644
--- a/crates/backend-uzu/src/forward_pass/model_shape.rs
+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs
@@ -3,7 +3,7 @@ use crate::{
config::{DecoderConfig, DecoderLayerType, LinearConfig, MLPConfig, MixerConfig},
};
-#[derive(Debug)]
+#[derive(Debug, Clone)]
pub struct ModelShape {
activation_type: DataType,
kv_cache_type: DataType,
diff --git a/crates/backend-uzu/src/forward_pass/scratch_buffers.rs b/crates/backend-uzu/src/forward_pass/scratch_buffers.rs
deleted file mode 100644
index 2cefb1d1c..000000000
--- a/crates/backend-uzu/src/forward_pass/scratch_buffers.rs
+++ /dev/null
@@ -1,216 +0,0 @@
-use crate::{
- DataType,
- array::{Array, ArrayContextExt},
- backends::common::Backend,
- config::{DecoderConfig, MLPConfig},
- forward_pass::model_shape::ModelShape,
-};
-
-pub struct ScratchBuffers<B: Backend> {
- // 1-D
- pub token_ids: Array<B>,
- pub token_subtrie_ranges: Array<B>,
- pub token_positions: Array<B>,
- pub token_parents: Array<B>,
- pub token_bitmask: Array<B>,
- pub token_seeds: Array<B>,
- pub sampling_output: Array<B>,
-
- // 2-D
- pub logits: Array<B>,
- pub main: Array<B>,
- pub shortcut: Array<B>,
- pub qkv: Array<B>,
- pub gate: Option<Array<B>>,
- pub attention_output: Array<B>,
- pub mlp_fused_up: Array<B>,
- pub mlp_hidden: Array<B>,
- pub lora_intermediate: Option<Array<B>>,
- pub ssm_inproj: Option<Array<B>>,
- pub ssm_packed: Option<Array<B>>,
- pub ssm_conv_padded: Option<Array<B>>,
- pub ssm_x: Option<Array<B>>,
- pub ssm_b: Option<Array<B>>,
- pub ssm_c: Option<Array<B>>,
- pub ssm_dt: Option<Array<B>>,
- pub ssm_z: Option<Array<B>>,
- pub delta_net_prep_q_norm: Option<Array<B>>,
- pub delta_net_prep_k_norm: Option<Array<B>>,
- pub delta_net_prep_beta: Option<Array<B>>,
- pub delta_net_prep_decay: Option<Array<B>>,
-
- // 3-D
- pub rotated_queries: Array<B>,
- pub rotated_keys: Array<B>,
- pub extracted_values: Array<B>,
-
- // 2-pass attention intermediate buffers
- pub attention_partials: Array<B>, // [num_heads * max_suffix_len * total_blocks_count * head_dim]
- pub attention_sums: Array<B>, // [num_heads * max_suffix_len * total_blocks_count]
- pub attention_maxs: Array<B>, // [num_heads * max_suffix_len * total_blocks_count]
-
- pub moe_topk_ids: Option<Array<B>>,
- pub moe_topk_probs: Option<Array<B>>,
- pub moe_offsets: Option<Array<B>>,
- pub moe_sumk: Option<Array<B>>,
- pub moe_bucketed_token_ids: Option<Array<B>>,
- pub moe_bucketed_probs: Option<Array<B>>,
- pub moe_x_perm: Option<Array<B>>,
- pub moe_tok2row: Option<Array<B>>,
- pub moe_y_partial: Option<Array<B>>,
- pub moe_hidden: Option<Array<B>>,
- pub moe_two_pass_row_expert_map: Option<Array<B>>,
- pub moe_tile_counts: Option<Array<B>>,
- pub moe_tile_offsets: Option<Array<B>>,
- pub moe_tile_map: Option<Array<B>>,
- pub moe_total_tiles: Option<Array<B>>,
- pub moe_dispatch_args: Option<Array<B>>,
- pub moe_scatter_partials: Option<Array<B>>,
- pub moe_scatter_block_bases: Option<Array<B>>,
- pub moe_block_alloc: Option<Array<B>>,
-}
-
-impl<B: Backend> ScratchBuffers<B> {
- pub fn new(
- context: &B::Context,
- decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
- max_suffix_len: usize,
- ) -> Self {
- // Helper closure for allocation
- let alloc = |shape: &[usize], dtype: DataType, label: &str| -> Array<B> {
- context.create_array_uninitialized(shape, dtype, &format!("scratch_buffers_{label}"))
- };
-
- let act_ty = model_shape.activation_data_type();
-
- let sums_maxs_shape = model_shape.attention_sums_shape(max_suffix_len);
- let moe = match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => Some(moe),
- _ => None,
- };
- let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_active_routed_experts);
- let moe_scatter_entries = moe.map(|moe| {
- let num_blocks = ((max_suffix_len + 255) / 256).max(1);
- let num_tiles = ((moe.num_routed_experts + 512 - 1) / 512).max(1);
- num_blocks * num_tiles * 512
- });
-
- Self {
- // 1-D
- token_ids: alloc(&[max_suffix_len], DataType::U64, "token_ids"),
- token_subtrie_ranges: alloc(
- &model_shape.subtrie_ranges_shape(max_suffix_len),
- DataType::U32,
- "token_subtrie_ranges",
- ),
- token_positions: alloc(&[max_suffix_len], DataType::I32, "token_positions"),
- token_parents: alloc(&[max_suffix_len], DataType::I32, "token_parents"),
- token_bitmask: alloc(&model_shape.bitmask_shape(max_suffix_len), DataType::U32, "token_bitmask"),
- token_seeds: alloc(&[max_suffix_len], DataType::U64, "token_seeds"),
- sampling_output: alloc(&[max_suffix_len], DataType::U32, "sampling_output"),
-
- // 2-D
- logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty, "logits"),
- main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),
- shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),
- qkv: alloc(&model_shape.qkv_shape(max_suffix_len), act_ty, "qkv"),
- gate: model_shape.gate_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "gate")),
- attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),
- mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),
- mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),
- lora_intermediate: model_shape
- .lora_intermediate(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "lora_intermediate")),
- ssm_inproj: model_shape.ssm_inproj_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_inproj")),
- ssm_packed: model_shape.ssm_packed_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_packed")),
- ssm_conv_padded: model_shape
- .ssm_conv_padded_shape(max_suffix_len)
- .map(|shape| alloc(&shape, act_ty, "ssm_conv_padded")),
- ssm_x: model_shape.ssm_x_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_x")),
- ssm_b: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_b")),
- ssm_c: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_c")),
- ssm_dt: model_shape.ssm_dt_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_dt")),
- ssm_z: model_shape.ssm_z_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_z")),
- delta_net_prep_q_norm: model_shape
- .delta_net_prep_qk_shape(max_suffix_len)
- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_q_norm")),
- delta_net_prep_k_norm: model_shape
- .delta_net_prep_qk_shape(max_suffix_len)
- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_k_norm")),
- delta_net_prep_beta: model_shape
- .delta_net_prep_beta_decay_shape(max_suffix_len)
- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_beta")),
- delta_net_prep_decay: model_shape
- .delta_net_prep_beta_decay_shape(max_suffix_len)
- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_decay")),
- // 3-D
- rotated_queries: alloc(&model_shape.rotated_queries_shape(max_suffix_len), act_ty, "rotated_queries"),
- rotated_keys: alloc(&model_shape.rotated_keys_shape(max_suffix_len), act_ty, "rotated_keys"),
- extracted_values: alloc(&model_shape.extracted_values_shape(max_suffix_len), act_ty, "extracted_values"),
-
- attention_partials: alloc(
- &model_shape.attention_partials_shape(max_suffix_len),
- act_ty,
- "attention_partials",
- ),
- attention_sums: alloc(&sums_maxs_shape, act_ty, "attention_sums"),
- attention_maxs: alloc(&sums_maxs_shape, act_ty, "attention_maxs"),
-
- moe_topk_ids: moe.map(|moe| {
- alloc(
- &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_active_routed_experts),
- DataType::U32,
- "moe_topk_ids",
- )
- }),
- moe_topk_probs: moe.map(|moe| {
- alloc(
- &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_active_routed_experts),
- act_ty,
- "moe_topk_probs",
- )
- }),
- moe_offsets: moe
- .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_offsets")),
- moe_sumk: moe.map(|_| alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")),
- moe_bucketed_token_ids: moe_max_routed.map(|max_routed| {
- alloc(&model_shape.moe_bucketed_token_ids_shape(max_routed), DataType::U32, "moe_bucketed_token_ids")
- }),
- moe_bucketed_probs: moe_max_routed.map(|max_routed| {
- alloc(&model_shape.moe_bucketed_probs_shape(max_routed), act_ty, "moe_bucketed_probs")
- }),
- moe_x_perm: moe_max_routed
- .map(|max_routed| alloc(&model_shape.moe_x_perm_shape(max_routed), DataType::F16, "moe_x_perm")),
- moe_tok2row: moe.map(|moe| {
- alloc(
- &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_active_routed_experts),
- DataType::I32,
- "moe_tok2row",
- )
- }),
- moe_y_partial: moe_max_routed
- .map(|max_routed| alloc(&model_shape.moe_y_partial_shape(max_routed), DataType::F16, "moe_y_partial")),
- moe_hidden: moe_max_routed
- .map(|max_routed| alloc(&model_shape.moe_hidden_shape(max_routed), DataType::F32, "moe_hidden")),
- moe_two_pass_row_expert_map: moe_max_routed
- .map(|max_routed| alloc(&[max_routed], DataType::U32, "moe_two_pass_row_expert_map")),
- moe_tile_counts: moe.map(|moe| {
- alloc(&model_shape.moe_counts_shape(moe.num_routed_experts), DataType::U32, "moe_tile_counts")
- }),
- moe_tile_offsets: moe.map(|moe| {
- alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_tile_offsets")
- }),
- moe_tile_map: moe_max_routed
- .map(|max_routed| alloc(&model_shape.moe_tile_map_shape(max_routed), DataType::U32, "moe_tile_map")),
- moe_total_tiles: moe.map(|_| alloc(&model_shape.moe_total_tiles_shape(), DataType::U32, "moe_total_tiles")),
- moe_dispatch_args: moe
- .map(|_| alloc(&model_shape.moe_dispatch_args_shape(), DataType::U32, "moe_dispatch_args")),
- moe_scatter_partials: moe_scatter_entries
- .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_partials")),
- moe_scatter_block_bases: moe_scatter_entries
- .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_block_bases")),
- moe_block_alloc: moe_scatter_entries.map(|entries| alloc(&[entries], DataType::U32, "moe_block_alloc")),
- }
- }
-}
diff --git a/crates/backend-uzu/src/forward_pass/short_conv_layer.rs b/crates/backend-uzu/src/forward_pass/short_conv_layer.rs
index a7f7958e7..dbb7474fc 100644
--- a/crates/backend-uzu/src/forward_pass/short_conv_layer.rs
+++ b/crates/backend-uzu/src/forward_pass/short_conv_layer.rs
@@ -1,16 +1,20 @@
use std::cell::Cell;
-use bytemuck::fill_zeroes;
+use crate::{
+ DataType,
+ array::size_for_shape,
+ backends::common::{Allocation, Backend, Encoder},
+};
-use crate::{array::Array, backends::common::Backend};
-
-#[derive(Debug)]
pub struct ShortConvLayer<B: Backend> {
- pub conv_state: Array<B>,
+ pub conv_state: Allocation<B>,
+ pub conv_shape: [usize; 2],
/// Per-token post-state written during speculative runs.
///
/// Shape: [max_suffix_length, model_dim, state_stride]
- pub suffix_state: Array<B>,
+ pub suffix_state: Allocation<B>,
+ pub suffix_shape: [usize; 3],
+ pub data_type: DataType,
/// Start index (in the suffix batch) for which `suffix_state` contains
/// valid post-states for this layer.
pub suffix_state_valid_start: Cell<usize>,
@@ -20,12 +24,6 @@ pub struct ShortConvLayer<B: Backend> {
}
impl<B: Backend> ShortConvLayer<B> {
- pub fn zero(&mut self) {
- fill_zeroes(self.conv_state.as_bytes_mut());
- fill_zeroes(self.suffix_state.as_bytes_mut());
- self.clear_suffix_state_valid_range();
- }
-
pub fn clear_suffix_state_valid_range(&self) {
self.suffix_state_valid_start.set(0);
self.suffix_state_valid_len.set(0);
@@ -43,6 +41,7 @@ impl<B: Backend> ShortConvLayer<B> {
pub fn commit_from_suffix_state_if_valid(
&mut self,
commit_index: usize,
+ encoder: &mut Encoder<B>,
) {
let start = self.suffix_state_valid_start.get();
let len = self.suffix_state_valid_len.get();
@@ -53,38 +52,19 @@ impl<B: Backend> ShortConvLayer<B> {
return;
}
- assert_eq!(
- self.conv_state.data_type(),
- self.suffix_state.data_type(),
- "ShortConv conv_state / suffix_state dtype mismatch"
- );
-
- let [model_dim, state_stride] = {
- let shape = self.conv_state.shape();
- assert_eq!(shape.len(), 2, "ShortConv conv_state expected 2-D [model_dim, state_stride], got {:?}", shape);
- [shape[0], shape[1]]
- };
+ let [model_dim, state_stride] = self.conv_shape;
+ let [suffix_length, suffix_model_dim, suffix_stride] = self.suffix_shape;
+ assert_eq!(suffix_model_dim, model_dim, "ShortConv suffix_state model_dim mismatch");
+ assert_eq!(suffix_stride, state_stride, "ShortConv suffix_state stride mismatch");
+ assert!(commit_index < suffix_length, "ShortConv commit_index {} out of range {}", commit_index, suffix_length);
- let suffix_shape = self.suffix_state.shape();
- assert!(
- suffix_shape.len() == 3 && suffix_shape[1] == model_dim && suffix_shape[2] == state_stride,
- "ShortConv suffix_state expected 3-D [suffix_len, model_dim, state_stride], got {:?}",
- suffix_shape
- );
- assert!(
- commit_index < suffix_shape[0],
- "ShortConv commit_index {} out of suffix_state range {}",
- commit_index,
- suffix_shape[0]
+ let bytes_per_token = size_for_shape(&[model_dim, state_stride], self.data_type);
+ let src_start = commit_index * bytes_per_token;
+ encoder.encode_copy(
+ &self.suffix_state,
+ src_start..src_start + bytes_per_token,
+ &mut self.conv_state,
+ 0..bytes_per_token,
);
-
- let elem_bytes = self.conv_state.data_type().size_in_bytes();
- let bytes_per_token = model_dim.saturating_mul(state_stride).saturating_mul(elem_bytes);
- let src_start = commit_index.saturating_mul(bytes_per_token);
- let src_end = src_start.saturating_add(bytes_per_token);
-
- let src = &self.suffix_state.as_bytes()[src_start..src_end];
- let dst = self.conv_state.as_bytes_mut();
- dst.copy_from_slice(src);
}
}
diff --git a/crates/backend-uzu/src/forward_pass/ssm_layer.rs b/crates/backend-uzu/src/forward_pass/ssm_layer.rs
index 807a65762..d7733d5d6 100644
--- a/crates/backend-uzu/src/forward_pass/ssm_layer.rs
+++ b/crates/backend-uzu/src/forward_pass/ssm_layer.rs
@@ -1,16 +1,12 @@
-use bytemuck::fill_zeroes;
+use crate::{
+ DataType,
+ backends::common::{Allocation, Backend},
+};
-use crate::{array::Array, backends::common::Backend};
-
-#[derive(Debug)]
pub struct SSMLayer<B: Backend> {
- pub conv_state: Array<B>,
- pub ssm_state: Array<B>,
-}
-
-impl<B: Backend> SSMLayer<B> {
- pub fn zero(&mut self) {
- fill_zeroes(self.conv_state.as_bytes_mut());
- fill_zeroes(self.ssm_state.as_bytes_mut());
- }
+ pub conv_state: Option<Allocation<B>>,
+ pub conv_shape: [usize; 2],
+ pub ssm_state: Allocation<B>,
+ pub ssm_shape: [usize; 3],
+ pub data_type: DataType,
}
diff --git a/crates/backend-uzu/src/forward_pass/state/array_id.rs b/crates/backend-uzu/src/forward_pass/state/array_id.rs
deleted file mode 100644
index 281b1470b..000000000
--- a/crates/backend-uzu/src/forward_pass/state/array_id.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use super::RopeType;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
-pub enum ArrayId {
- TokenIds,
- TokenPositions,
- TokenParents,
- TokenBitmask,
- Logits,
- TokenSeeds,
-
- Main,
- Shortcut,
- QKV,
- Gate,
- AttentionOutput,
- MlpFusedUp,
- MlpHidden,
- SsmInProj,
-
- Keys(usize),
- Values(usize),
- SsmConvState(usize),
- SsmState(usize),
- SsmPacked(usize),
- SsmX(usize),
- SsmB(usize),
- SsmC(usize),
- SsmDt(usize),
- SsmZ(usize),
- ShortConvState(usize),
- ShortConvSuffixState(usize),
- DeltaNetConvState(usize),
- DeltaNetSsmState(usize),
-
- RotatedQueries,
- RotatedKeys,
- ExtractedValues,
-
- AttentionPartials,
- AttentionSums,
- AttentionMaxs,
-
- RopeCosines(RopeType),
- RopeSines(RopeType),
-
- AttentionSinks(usize),
-
- MoeTopkIds,
- MoeTopkProbs,
- MoeOffsets,
- MoeSumK,
- MoeBucketedTokenIds,
- MoeBucketedProbs,
- MoeXPerm,
- MoeTok2Row,
- MoeYPartial,
- MoeHidden,
- MoeTwoPassRowExpertMap,
- MoeTileCounts,
- MoeTileOffsets,
- MoeTileMap,
- MoeTotalTiles,
- MoeDispatchArgs,
- MoeScatterPartials,
- MoeScatterBlockBases,
- MoeBlockAlloc,
-
- // Classifier prediction head buffers
- ClassifierPooling,
- ClassifierPredictionHeadDense,
- ClassifierPredictionHeadNorm,
- ClassifierPredictionHeadLogits,
-}
diff --git a/crates/backend-uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/backend-uzu/src/forward_pass/state/common_aux_buffers.rs
deleted file mode 100644
index 2124f53a6..000000000
--- a/crates/backend-uzu/src/forward_pass/state/common_aux_buffers.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-use crate::{
- array::Array,
- backends::common::Backend,
- forward_pass::{model_shape::ModelShape, scratch_buffers::ScratchBuffers},
-};
-
-pub struct CommonAuxBuffers<B: Backend> {
- pub suffix_length: usize,
- pub main: Array<B>,
- pub shortcut: Array<B>,
- pub qkv: Array<B>,
- pub gate: Option<Array<B>>,
- pub attention_output: Array<B>,
- pub mlp_fused_up: Array<B>,
- pub mlp_hidden: Array<B>,
- pub lora_intermediate: Option<Array<B>>,
- pub rotated_queries: Array<B>,
- pub rotated_keys: Array<B>,
- pub extracted_values: Array<B>,
- pub attention_partials: Array<B>,
- pub attention_sums: Array<B>,
- pub attention_maxs: Array<B>,
-}
-
-impl<B: Backend> CommonAuxBuffers<B> {
- pub fn new(
- scratch: &ScratchBuffers<B>,
- model_shape: &ModelShape,
- suffix_length: usize,
- ) -> Self {
- Self {
- suffix_length,
- main: scratch.main.view(&model_shape.main_shape(suffix_length)),
- shortcut: scratch.shortcut.view(&model_shape.main_shape(suffix_length)),
- qkv: scratch.qkv.view(&model_shape.qkv_shape(suffix_length)),
- gate: scratch.gate.as_ref().zip(model_shape.gate_shape(suffix_length)).map(|(buf, shape)| buf.view(&shape)),
- attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),
- mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),
- mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),
- lora_intermediate: scratch
- .lora_intermediate
- .as_ref()
- .zip(model_shape.lora_intermediate(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- rotated_queries: scratch.rotated_queries.view(&model_shape.rotated_queries_shape(suffix_length)),
- rotated_keys: scratch.rotated_keys.view(&model_shape.rotated_keys_shape(suffix_length)),
- extracted_values: scratch.extracted_values.view(&model_shape.extracted_values_shape(suffix_length)),
- attention_partials: scratch.attention_partials.view(&model_shape.attention_partials_shape(suffix_length)),
- attention_sums: scratch.attention_sums.view(&model_shape.attention_sums_shape(suffix_length)),
- attention_maxs: scratch.attention_maxs.view(&model_shape.attention_maxs_shape(suffix_length)),
- }
- }
-}
diff --git a/crates/backend-uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/backend-uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
deleted file mode 100644
index 09a974544..000000000
--- a/crates/backend-uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
+++ /dev/null
@@ -1,178 +0,0 @@
-use crate::{
- array::Array,
- backends::common::Backend,
- config::{DecoderConfig, MLPConfig},
- forward_pass::{model_shape::ModelShape, scratch_buffers::ScratchBuffers},
-};
-
-pub struct LanguageModelGeneratorAuxBuffers<B: Backend> {
- pub ssm_inproj: Option<Array<B>>,
- pub ssm_packed: Option<Array<B>>,
- pub ssm_conv_padded: Option<Array<B>>,
- pub ssm_x: Option<Array<B>>,
- pub ssm_b: Option<Array<B>>,
- pub ssm_c: Option<Array<B>>,
- pub ssm_dt: Option<Array<B>>,
- pub ssm_z: Option<Array<B>>,
- // DeltaNet prep buffers
- pub delta_net_prep_q_norm: Option<Array<B>>,
- pub delta_net_prep_k_norm: Option<Array<B>>,
- pub delta_net_prep_beta: Option<Array<B>>,
- pub delta_net_prep_decay: Option<Array<B>>,
- // MoE buffers
- pub moe_topk_ids: Option<Array<B>>,
- pub moe_topk_probs: Option<Array<B>>,
- pub moe_offsets: Option<Array<B>>,
- pub moe_sumk: Option<Array<B>>,
- pub moe_bucketed_token_ids: Option<Array<B>>,
- pub moe_bucketed_probs: Option<Array<B>>,
- pub moe_x_perm: Option<Array<B>>,
- pub moe_tok2row: Option<Array<B>>,
- pub moe_y_partial: Option<Array<B>>,
- pub moe_hidden: Option<Array<B>>,
- pub moe_two_pass_row_expert_map: Option<Array<B>>,
- pub moe_tile_counts: Option<Array<B>>,
- pub moe_tile_offsets: Option<Array<B>>,
- pub moe_tile_map: Option<Array<B>>,
- pub moe_total_tiles: Option<Array<B>>,
- pub moe_dispatch_args: Option<Array<B>>,
- pub moe_scatter_partials: Option<Array<B>>,
- pub moe_scatter_block_bases: Option<Array<B>>,
- pub moe_block_alloc: Option<Array<B>>,
-}
-
-impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
- pub fn new(
- scratch: &ScratchBuffers<B>,
- decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
- suffix_length: usize,
- ) -> Self {
- let moe = match &decoder_config.layer_config.mlp_config {
- MLPConfig::MixtureOfExperts(moe) => Some(moe),
- _ => None,
- };
- let max_routed = moe.map(|moe| suffix_length * moe.num_active_routed_experts);
- let num_blocks = suffix_length.div_ceil(256).max(1);
- let scatter_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1) * 512);
- let block_alloc_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1));
-
- Self {
- ssm_inproj: scratch
- .ssm_inproj
- .as_ref()
- .zip(model_shape.ssm_inproj_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- ssm_packed: scratch
- .ssm_packed
- .as_ref()
- .zip(model_shape.ssm_packed_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- ssm_conv_padded: scratch
- .ssm_conv_padded
- .as_ref()
- .zip(model_shape.ssm_conv_padded_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- ssm_x: scratch
- .ssm_x
- .as_ref()
- .zip(model_shape.ssm_x_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- ssm_b: scratch
- .ssm_b
- .as_ref()
- .zip(model_shape.ssm_bc_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- ssm_c: scratch
- .ssm_c
- .as_ref()
- .zip(model_shape.ssm_bc_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- ssm_dt: scratch
- .ssm_dt
- .as_ref()
- .zip(model_shape.ssm_dt_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- ssm_z: scratch
- .ssm_z
- .as_ref()
- .zip(model_shape.ssm_z_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- delta_net_prep_q_norm: scratch
- .delta_net_prep_q_norm
- .as_ref()
- .zip(model_shape.delta_net_prep_qk_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- delta_net_prep_k_norm: scratch
- .delta_net_prep_k_norm
- .as_ref()
- .zip(model_shape.delta_net_prep_qk_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- delta_net_prep_beta: scratch
- .delta_net_prep_beta
- .as_ref()
- .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- delta_net_prep_decay: scratch
- .delta_net_prep_decay
- .as_ref()
- .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))
- .map(|(buf, shape)| buf.view(&shape)),
- moe_topk_ids: moe.zip(scratch.moe_topk_ids.as_ref()).map(|(moe, buf)| {
- buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_active_routed_experts))
- }),
- moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(|(moe, buf)| {
- buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_active_routed_experts))
- }),
- moe_offsets: moe
- .zip(scratch.moe_offsets.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),
- moe_sumk: moe.and(scratch.moe_sumk.as_ref()).map(|buf| buf.view(&model_shape.moe_sumk_shape())),
- moe_bucketed_token_ids: max_routed
- .zip(scratch.moe_bucketed_token_ids.as_ref())
- .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_token_ids_shape(max_routed))),
- moe_bucketed_probs: max_routed
- .zip(scratch.moe_bucketed_probs.as_ref())
- .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_probs_shape(max_routed))),
- moe_x_perm: max_routed
- .zip(scratch.moe_x_perm.as_ref())
- .map(|(max_routed, buf)| buf.view(&model_shape.moe_x_perm_shape(max_routed))),
- moe_tok2row: moe.zip(scratch.moe_tok2row.as_ref()).map(|(moe, buf)| {
- buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_active_routed_experts))
- }),
- moe_y_partial: max_routed
- .zip(scratch.moe_y_partial.as_ref())
- .map(|(max_routed, buf)| buf.view(&model_shape.moe_y_partial_shape(max_routed))),
- moe_hidden: max_routed
- .zip(scratch.moe_hidden.as_ref())
- .map(|(max_routed, buf)| buf.view(&model_shape.moe_hidden_shape(max_routed))),
- moe_two_pass_row_expert_map: max_routed
- .zip(scratch.moe_two_pass_row_expert_map.as_ref())
- .map(|(max_routed, buf)| buf.view(&[max_routed])),
- moe_tile_counts: moe
- .zip(scratch.moe_tile_counts.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.num_routed_experts))),
- moe_tile_offsets: moe
- .zip(scratch.moe_tile_offsets.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),
- moe_tile_map: max_routed
- .zip(scratch.moe_tile_map.as_ref())
- .map(|(max_routed, buf)| buf.view(&model_shape.moe_tile_map_shape(max_routed))),
- moe_total_tiles: moe
- .and(scratch.moe_total_tiles.as_ref())
- .map(|buf| buf.view(&model_shape.moe_total_tiles_shape())),
- moe_dispatch_args: moe
- .and(scratch.moe_dispatch_args.as_ref())
- .map(|buf| buf.view(&model_shape.moe_dispatch_args_shape())),
- moe_scatter_partials: scatter_entries
- .zip(scratch.moe_scatter_partials.as_ref())
- .map(|(entries, buf)| buf.view(&[entries])),
- moe_scatter_block_bases: scatter_entries
- .zip(scratch.moe_scatter_block_bases.as_ref())
- .map(|(entries, buf)| buf.view(&[entries])),
- moe_block_alloc: block_alloc_entries
- .zip(scratch.moe_block_alloc.as_ref())
- .map(|(entries, buf)| buf.view(&[entries])),
- }
- }
-}
diff --git a/crates/backend-uzu/src/forward_pass/state/mod.rs b/crates/backend-uzu/src/forward_pass/state/mod.rs
index 531093517..efa4164c8 100644
--- a/crates/backend-uzu/src/forward_pass/state/mod.rs
+++ b/crates/backend-uzu/src/forward_pass/state/mod.rs
@@ -1,17 +1,8 @@
-mod array_id;
mod rope_type;
-mod common_aux_buffers;
-mod language_model_generator_aux_buffers;
mod rope_buffers;
mod shared_buffers;
-mod state;
-
-pub use array_id::ArrayId;
-pub use common_aux_buffers::CommonAuxBuffers;
-pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;
pub use rope_buffers::RopeBuffers;
pub use rope_type::RopeType;
pub use shared_buffers::SharedBuffers;
-pub use state::ForwardPassState;
diff --git a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
index 39de3ab9c..37894d4a7 100644
--- a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
+++ b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs
@@ -1,15 +1,15 @@
use crate::{
- array::{Array, ArrayContextExt},
- backends::common::Backend,
+ array::ArrayContextExt,
+ backends::common::{Allocation, Backend},
forward_pass::model_shape::ModelShape,
parameters::ParameterTree,
};
pub struct RopeBuffers<B: Backend> {
/// [rope_max_sequence_length, rope_dim]
- pub cosines: Array<B>,
+ pub cosines: Allocation<B>,
/// [rope_max_sequence_length, rope_dim]
- pub sines: Array<B>,
+ pub sines: Allocation<B>,
}
impl<B: Backend> RopeBuffers<B> {
@@ -21,16 +21,12 @@ impl<B: Backend> RopeBuffers<B> {
let rope_max_sequence_length = model_shape.context_length();
Self {
- cosines: context.create_array_uninitialized(
- &[rope_max_sequence_length, rope_dim],
- model_shape.activation_data_type(),
- "rope_buffers_cosines",
- ),
- sines: context.create_array_uninitialized(
- &[rope_max_sequence_length, rope_dim],
- model_shape.activation_data_type(),
- "rope_buffers_sines",
- ),
+ cosines: context
+ .create_array_uninitialized(&[rope_max_sequence_length, rope_dim], model_shape.activation_data_type())
+ .into_allocation(),
+ sines: context
+ .create_array_uninitialized(&[rope_max_sequence_length, rope_dim], model_shape.activation_data_type())
+ .into_allocation(),
}
}
@@ -43,10 +39,7 @@ impl<B: Backend> RopeBuffers<B> {
return;
};
- let cosines_view = rope_tree.leaf_array("cosines").unwrap();
- self.cosines.copy_from_array(&cosines_view);
-
- let sines_view = rope_tree.leaf_array("sines").unwrap();
- self.sines.copy_from_array(&sines_view);
+ self.cosines = rope_tree.leaf_allocation("cosines").unwrap();
+ self.sines = rope_tree.leaf_allocation("sines").unwrap();
}
}
diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
index 2e9e8a49b..6011bbd2c 100644
--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
+++ b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
@@ -1,10 +1,10 @@
use half::{bf16, f16};
-use super::RopeBuffers;
+use super::{RopeBuffers, RopeType};
use crate::{
DataType,
- array::{Array, ArrayContextExt},
- backends::common::Backend,
+ array::ArrayContextExt,
+ backends::common::{Allocation, AsBufferRangeMut, Backend, DenseBuffer},
config::DecoderConfig,
forward_pass::model_shape::ModelShape,
parameters::ParameterTree,
@@ -13,7 +13,7 @@ use crate::{
pub struct SharedBuffers<B: Backend> {
pub global_rope: Option<RopeBuffers<B>>,
pub local_rope: Option<RopeBuffers<B>>,
- pub attention_sinks: Option<Vec<Array<B>>>,
+ pub attention_sinks: Option<Vec<Allocation<B>>>,
}
impl<B: Backend> SharedBuffers<B> {
@@ -29,9 +29,7 @@ impl<B: Backend> SharedBuffers<B> {
let attention_sinks = decoder_config.layer_config.attention_config().is_some_and(|c| c.has_sinks).then(|| {
let num_heads = decoder_config.num_heads;
(0..decoder_config.num_layers)
- .map(|_| {
- context.create_array_uninitialized(&[num_heads], DataType::F32, "shared_buffers_attention_sinks")
- })
+ .map(|_| context.create_array_uninitialized(&[num_heads], DataType::F32).into_allocation())
.collect()
});
@@ -60,7 +58,14 @@ impl<B: Backend> SharedBuffers<B> {
let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();
let attn_tree = layer_tree.subtree("mixer").unwrap();
let sinks_arr = attn_tree.leaf_array("sinks").unwrap();
- let dst_slice = sink_cell.as_slice_mut::<f32>();
+ let dst_slice = unsafe {
+ let buffer_range = sink_cell.as_buffer_range_mut();
+ let range = buffer_range.range();
+ std::slice::from_raw_parts_mut(
+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start) as *mut f32,
+ range.len() / std::mem::size_of::<f32>(),
+ )
+ };
match sinks_arr.data_type() {
DataType::F32 => {
@@ -86,4 +91,31 @@ impl<B: Backend> SharedBuffers<B> {
}
}
}
+
+ pub fn rope_cosines(
+ &self,
+ rope_type: RopeType,
+ ) -> Option<&Allocation<B>> {
+ match rope_type {
+ RopeType::Global => self.global_rope.as_ref().map(|rope| &rope.cosines),
+ RopeType::Local => self.local_rope.as_ref().map(|rope| &rope.cosines),
+ }
+ }
+
+ pub fn rope_sines(
+ &self,
+ rope_type: RopeType,
+ ) -> Option<&Allocation<B>> {
+ match rope_type {
+ RopeType::Global => self.global_rope.as_ref().map(|rope| &rope.sines),
+ RopeType::Local => self.local_rope.as_ref().map(|rope| &rope.sines),
+ }
+ }
+
+ pub fn attention_sinks(
+ &self,
+ layer_index: usize,
+ ) -> Option<&Allocation<B>> {
+ self.attention_sinks.as_ref().map(|sinks| &sinks[layer_index])
+ }
}
diff --git a/crates/backend-uzu/src/forward_pass/state/state.rs b/crates/backend-uzu/src/forward_pass/state/state.rs
deleted file mode 100644
index 9d3c96645..000000000
--- a/crates/backend-uzu/src/forward_pass/state/state.rs
+++ /dev/null
@@ -1,698 +0,0 @@
-#[cfg(feature = "tracing")]
-use std::ops::{Deref, DerefMut};
-use std::{cell::RefCell, rc::Rc};
-
-use ndarray::ArrayView2;
-
-#[cfg(feature = "tracing")]
-use crate::backends::common::Encoder;
-#[cfg(feature = "tracing")]
-use crate::forward_pass::traces::ActivationTrace;
-use crate::{
- DataType,
- array::Array,
- backends::common::{Backend, Context},
- config::DecoderConfig,
- forward_pass::{
- cache_layers::CacheLayers,
- model_shape::ModelShape,
- scratch_buffers::ScratchBuffers,
- state::{ArrayId, CommonAuxBuffers, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},
- },
- session::parameter::SamplingMethod,
-};
-
-pub enum ForwardPassMode<B: Backend> {
- LanguageModelGenerator(LanguageModelGeneratorModeState<B>),
- Classifier(ClassifierModeState<B>),
-}
-
-pub struct LanguageModelGeneratorModeState<B: Backend> {
- pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
- pub token_seeds: Array<B>,
- pub logits: Array<B>,
- pub sampling_output: Option<Array<B>>,
- pub sampling_method: Option<SamplingMethod>,
- #[cfg(feature = "tracing")]
- pub traces: Rc<RefCell<ActivationTrace<B>>>,
- pub active_row_count: usize,
- pub sampling_start: usize,
- pub sampling_length: usize,
- pub is_prefilling: bool,
-}
-
-pub struct ClassifierModeState<B: Backend> {
- pub pooling: Array<B>,
- pub dense: Array<B>,
- pub norm: Array<B>,
- pub classifier_logits: Array<B>,
- pub active_row_count: usize,
- #[cfg(feature = "tracing")]
- pub traces: Rc<RefCell<ActivationTrace<B>>>,
-}
-
-pub struct ForwardPassState<B: Backend> {
- context: Rc<B::Context>,
- token_ids: Array<B>,
- pub token_subtrie_ranges: Option<Array<B>>,
- token_positions: Array<B>,
- token_parents: Array<B>,
- token_bitmask: Option<Array<B>>,
- pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- pub common_aux: CommonAuxBuffers<B>,
- pub llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,
- mode: ForwardPassMode<B>,
-}
-
-impl<B: Backend> ForwardPassState<B> {
- // ========================================================================
- // Common initialization helpers
- // ========================================================================
-
- fn init_token_ids(
- scratch: &ScratchBuffers<B>,
- token_ids: &[u64],
- ) -> Array<B> {
- let suffix_length = token_ids.len();
- let mut token_ids_array = scratch.token_ids.view(&[suffix_length]);
- token_ids_array.copy_from_view(token_ids.into());
- token_ids_array
- }
-
- fn init_token_positions(
- scratch: &ScratchBuffers<B>,
- token_positions: &[usize],
- ) -> Array<B> {
- let suffix_length = token_positions.len();
- let mut token_positions_array = scratch.token_positions.view(&[suffix_length]);
- let token_positions_i32: Box<[i32]> = token_positions.iter().map(|p| *p as i32).collect();
- token_positions_array.copy_from_view(token_positions_i32.as_ref().into());
- token_positions_array
- }
-
- fn init_token_parents(
- scratch: &ScratchBuffers<B>,
- token_positions: &[usize],
- sampling_start: usize,
- sampling_length: usize,
- ) -> Array<B> {
- let suffix_length = token_positions.len();
- let mut token_parents_array = scratch.token_parents.view(&[suffix_length]);
- {
- let parents = token_parents_array.as_slice_mut::<i32>();
- parents.fill(-1);
-
- if sampling_length > 0 {
- let root_pos = token_positions.get(sampling_start).copied().unwrap_or(0);
-
- let mut stack: Vec<usize> = Vec::new();
-
- for local_idx in 0..sampling_length {
- let abs_idx = sampling_start + local_idx;
- let Some(&pos) = token_positions.get(abs_idx) else {
- break;
- };
- let depth = pos.saturating_sub(root_pos);
-
- let parent_local: i32 = if depth == 0 {
- -1
- } else if let Some(&p) = stack.get(depth - 1) {
- p as i32
- } else {
- debug_assert!(false, "invalid trie depth ordering: depth={depth}, stack_len={}", stack.len());
- -1
- };
- parents[abs_idx] = parent_local;
-
- if stack.len() <= depth {
- stack.resize(depth + 1, 0);
- }
- stack[depth] = local_idx;
- stack.truncate(depth + 1);
- }
- }
- }
- token_parents_array
- }
-
- // ========================================================================
- // LLM Constructor
- // ========================================================================
-
- #[allow(clippy::too_many_arguments)]
- pub fn new_llm(
- context: Rc<B::Context>,
- decoder_config: &DecoderConfig,
- model_shape: &ModelShape,
- scratch: &ScratchBuffers<B>,
- cache_layers: Rc<RefCell<CacheLayers<B>>>,
- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- token_ids: &[u64],
- token_subtrie_ranges: Option<&[[u32; 3]]>,
- token_positions: &[usize],
- token_bitmask: Option<&[u32]>,
- token_seeds: &[u64],
- active_row_count: usize,
- sampling_start: usize,
- sampling_length: usize,
- is_prefilling: bool,
- skip_token_ids_copy: bool,
- async_positions: Option<(Rc<RefCell<B::DenseBuffer>>, usize)>,
- async_seeds: Option<(Rc<RefCell<B::DenseBuffer>>, usize)>,
- ) -> Self {
- let suffix_length = token_ids.len();
- assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");
- assert!(suffix_length <= cache_layers.borrow().max_suffix_length(), "Suffix length exceeds KV cache capacity");
-
- // Token IDs - optionally skip copy for async path
- let token_ids_cell = if skip_token_ids_copy {
- scratch.token_ids.view(&[suffix_length])
- } else {
- Self::init_token_ids(scratch, token_ids)
- };
-
- // Token subtrie ranges
- let token_subtrie_ranges_cell = token_subtrie_ranges.map(|subtrie_ranges| {
- let subtrie_ranges_shape = model_shape.subtrie_ranges_shape(suffix_length);
- let mut subtrie_ranges_array = scratch.token_subtrie_ranges.view(&subtrie_ranges_shape);
- subtrie_ranges_array.as_slice_mut::<u32>().fill(0);
- subtrie_ranges_array.copy_from_view(
- ArrayView2::from_shape((subtrie_ranges.len(), 3), bytemuck::cast_slice::<_, u32>(subtrie_ranges))
- .unwrap(),
- );
- subtrie_ranges_array
- });
-
- // Token positions - use async buffer if provided
- let token_positions_cell = if let Some((async_buf, offset)) = async_positions {
- unsafe {
- Array::from_parts(async_buf, offset * std::mem::size_of::<i32>(), &[suffix_length], DataType::I32)
- }
- } else {
- Self::init_token_positions(scratch, token_positions)
- };
-
- // Trie parent indices (relative, within the sampling segment).
- // Only meaningful when sampling_length > 0.
- let token_parents_cell = Self::init_token_parents(scratch, token_positions, sampling_start, sampling_length);
-
- // Token bitmask
- let token_bitmask_cell = token_bitmask.map(|bitmask| {
- let bitmask_shape = model_shape.bitmask_shape(suffix_length);
- let mut bitmask_array = scratch.token_bitmask.view(&bitmask_shape);
- bitmask_array.as_slice_mut::<u32>().fill(0);
- bitmask_array.copy_from_view(bitmask.into());
- bitmask_array
- });
-
- // Token seeds - use async buffer if provided
- let token_seeds_cell = if let Some((async_buf, offset)) = async_seeds {
- unsafe {
- Array::from_parts(async_buf, offset * std::mem::size_of::<u64>(), &[suffix_length], DataType::U64)
- }
- } else {
- let mut token_seeds_array = scratch.token_seeds.view(&[suffix_length]);
- token_seeds_array.copy_from_view(token_seeds.into());
- token_seeds_array
- };
-
- // Logits
- let logits_cell = scratch.logits.view(&model_shape.logits_shape(suffix_length));
-
- // Sampling output
- let sampling_output = Some(scratch.sampling_output.view(&[suffix_length]));
-
- // Common aux buffers
- let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);
-
- // LLM-specific aux buffers
- let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(scratch, decoder_config, model_shape, suffix_length));
-
- // Traces
- #[cfg(feature = "tracing")]
- let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(context.as_ref(), model_shape, suffix_length)));
-
- let mode = ForwardPassMode::LanguageModelGenerator(LanguageModelGeneratorModeState {
- cache_layers,
- token_seeds: token_seeds_cell,
- logits: logits_cell,
- sampling_output,
- sampling_method: None,
- #[cfg(feature = "tracing")]
- traces,
- active_row_count,
- sampling_start,
- sampling_length,
- is_prefilling,
- });
- Self {
- context,
- token_ids: token_ids_cell,
- token_subtrie_ranges: token_subtrie_ranges_cell,
- token_positions: token_positions_cell,
- token_parents: token_parents_cell,
- token_bitmask: token_bitmask_cell,
- shared_buffers,
- common_aux,
- llm_aux,
- mode,
- }
- }
-
- // ========================================================================
- // Classifier Constructor
- // ========================================================================
-
- #[allow(clippy::too_many_arguments)]
- pub fn new_classifier(
- context: Rc<B::Context>,
- model_shape: &ModelShape,
- scratch: &ScratchBuffers<B>,
- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- token_ids: &[u64],
- token_positions: &[usize],
- num_labels: usize,
- ) -> Self {
- let suffix_length = token_ids.len();
- assert_eq!(suffix_length, token_positions.len());
-
- let token_ids_cell = Self::init_token_ids(scratch, token_ids);
- let token_positions_cell = Self::init_token_positions(scratch, token_positions);
- let token_parents_cell = Self::init_token_parents(scratch, token_positions, 0, 0);
-
- // Common aux buffers
- let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);
-
- // Classifier-specific buffers
- let model_dim = model_shape.main_shape(1)[1];
- let classifier_state =
- Self::init_classifier_buffers(context.as_ref(), model_shape, model_dim, num_labels, suffix_length);
-
- let mode = ForwardPassMode::Classifier(classifier_state);
-
- Self {
- context,
- token_ids: token_ids_cell,
- token_subtrie_ranges: None,
- token_positions: token_positions_cell,
- token_parents: token_parents_cell,
- token_bitmask: None,
- shared_buffers,
- common_aux,
- llm_aux: None,
- mode,
- }
- }
-
- #[cfg(feature = "tracing")]
- fn init_classifier_buffers(
- context: &B::Context,
- model_shape: &ModelShape,
- model_dim: usize,
- num_labels: usize,
- suffix_length: usize,
- ) -> ClassifierModeState<B> {
- let data_type = model_shape.activation_data_type();
- let batch_dim = 1;
-
- let create_buffer = |size: usize| -> Array<B> {
- let buffer_size = size * data_type.size_in_bytes();
- let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, size / batch_dim], data_type) }
- };
-
- ClassifierModeState {
- pooling: create_buffer(batch_dim * model_dim),
- dense: create_buffer(batch_dim * model_dim),
- norm: create_buffer(batch_dim * model_dim),
- classifier_logits: {
- let buffer_size = batch_dim * num_labels * data_type.size_in_bytes();
- let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, num_labels], data_type) }
- },
- active_row_count: suffix_length,
- traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(
- context,
- model_shape,
- suffix_length,
- num_labels,
- ))),
- }
- }
-
- #[cfg(not(feature = "tracing"))]
- fn init_classifier_buffers(
- context: &B::Context,
- model_shape: &ModelShape,
- model_dim: usize,
- num_labels: usize,
- suffix_length: usize,
- ) -> ClassifierModeState<B> {
- let data_type = model_shape.activation_data_type();
- let batch_dim = 1;
-
- let create_buffer = |dims: &[usize]| -> Array<B> {
- let size: usize = dims.iter().product();
- let buffer_size = size * data_type.size_in_bytes();
- let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, dims, data_type) }
- };
-
- ClassifierModeState {
- pooling: create_buffer(&[batch_dim, model_dim]),
- dense: create_buffer(&[batch_dim, model_dim]),
- norm: create_buffer(&[batch_dim, model_dim]),
- classifier_logits: create_buffer(&[batch_dim, num_labels]),
- active_row_count: suffix_length,
- }
- }
-
- // ========================================================================
- // Accessors
- // ========================================================================
-
- pub fn context(&self) -> &B::Context {
- self.context.as_ref()
- }
-
- pub fn aux_buffers_suffix_length(&self) -> usize {
- self.common_aux.suffix_length
- }
-
- pub fn token_bitmask(&self) -> Option<&Array<B>> {
- self.token_bitmask.as_ref()
- }
-
- pub fn llm_state(&self) -> &LanguageModelGeneratorModeState<B> {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state,
- _ => panic!("Not in LLM mode"),
- }
- }
-
- pub fn classifier_state(&self) -> &ClassifierModeState<B> {
- match &self.mode {
- ForwardPassMode::Classifier(state) => state,
- _ => panic!("Not in classifier mode"),
- }
- }
-
- pub fn array(
- &self,
- id: ArrayId,
- ) -> Array<B> {
- match id {
- // Common arrays
- ArrayId::TokenIds => self.token_ids.clone(),
- ArrayId::TokenPositions => self.token_positions.clone(),
- ArrayId::TokenParents => self.token_parents.clone(),
- ArrayId::TokenBitmask => self.token_bitmask.clone().expect("Token bitmask not available"),
- ArrayId::Main => self.common_aux.main.clone(),
- ArrayId::Shortcut => self.common_aux.shortcut.clone(),
- ArrayId::QKV => self.common_aux.qkv.clone(),
- ArrayId::Gate => self.common_aux.gate.clone().expect("Gate buffer not available"),
- ArrayId::AttentionOutput => self.common_aux.attention_output.clone(),
- ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),
- ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),
- ArrayId::RotatedQueries => self.common_aux.rotated_queries.clone(),
- ArrayId::RotatedKeys => self.common_aux.rotated_keys.clone(),
- ArrayId::ExtractedValues => self.common_aux.extracted_values.clone(),
- ArrayId::AttentionPartials => self.common_aux.attention_partials.clone(),
- ArrayId::AttentionSums => self.common_aux.attention_sums.clone(),
- ArrayId::AttentionMaxs => self.common_aux.attention_maxs.clone(),
-
- // Shared buffer arrays
- ArrayId::RopeCosines(_) | ArrayId::RopeSines(_) => {
- self.shared_buffer_array(id).expect("Shared buffer array should be available")
- },
-
- // LLM-specific arrays
- ArrayId::Logits => self.llm_state().logits.clone(),
- ArrayId::TokenSeeds => self.llm_state().token_seeds.clone(),
- ArrayId::Keys(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_transformer().expect("Expected transformer layer").keys.clone()
- },
- ArrayId::Values(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_transformer().expect("Expected transformer layer").values.clone()
- },
- ArrayId::AttentionSinks(layer_index) => {
- self.shared_buffers.borrow().attention_sinks.as_ref().expect("Attention sinks not initialized")
- [layer_index]
- .clone()
- },
-
- // SSM arrays (LLM only)
- ArrayId::SsmInProj => {
- self.llm_aux.as_ref().and_then(|aux| aux.ssm_inproj.clone()).expect("SSM inproj not initialized")
- },
- ArrayId::SsmPacked(_) => {
- self.llm_aux.as_ref().and_then(|aux| aux.ssm_packed.clone()).expect("SSM packed not initialized")
- },
- ArrayId::SsmConvState(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_state_space().expect("Expected SSM layer").conv_state.clone()
- },
- ArrayId::SsmState(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_state_space().expect("Expected SSM layer").ssm_state.clone()
- },
- ArrayId::SsmX(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_x.clone()).expect("SSM x not initialized"),
- ArrayId::SsmB(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_b.clone()).expect("SSM b not initialized"),
- ArrayId::SsmC(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_c.clone()).expect("SSM c not initialized"),
- ArrayId::SsmDt(_) => {
- self.llm_aux.as_ref().and_then(|aux| aux.ssm_dt.clone()).expect("SSM dt not initialized")
- },
- ArrayId::SsmZ(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_z.clone()).expect("SSM z not initialized"),
- ArrayId::ShortConvState(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").conv_state.clone()
- },
- ArrayId::ShortConvSuffixState(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").suffix_state.clone()
- },
- ArrayId::DeltaNetConvState(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").conv_state.clone()
- },
- ArrayId::DeltaNetSsmState(layer_index) => {
- let cache = self.llm_state().cache_layers.borrow();
- cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").ssm_state.clone()
- },
-
- // MoE arrays (LLM only)
- ArrayId::MoeTopkIds => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_topk_ids.clone()).expect("MoE topk_ids not initialized")
- },
- ArrayId::MoeTopkProbs => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_topk_probs.clone())
- .expect("MoE topk_probs not initialized"),
- ArrayId::MoeOffsets => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_offsets.clone()).expect("MoE offsets not initialized")
- },
- ArrayId::MoeSumK => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_sumk.clone()).expect("MoE sumk not initialized")
- },
- ArrayId::MoeBucketedTokenIds => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_bucketed_token_ids.clone())
- .expect("MoE bucketed_token_ids not initialized"),
- ArrayId::MoeBucketedProbs => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_bucketed_probs.clone())
- .expect("MoE bucketed_probs not initialized"),
- ArrayId::MoeXPerm => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_x_perm.clone()).expect("MoE x_perm not initialized")
- },
- ArrayId::MoeTok2Row => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_tok2row.clone()).expect("MoE tok2row not initialized")
- },
- ArrayId::MoeYPartial => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_y_partial.clone()).expect("MoE y_partial not initialized")
- },
- ArrayId::MoeHidden => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_hidden.clone()).expect("MoE hidden not initialized")
- },
- ArrayId::MoeTwoPassRowExpertMap => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_two_pass_row_expert_map.clone())
- .expect("MoE two_pass_row_expert_map not initialized"),
- ArrayId::MoeTileCounts => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_tile_counts.clone())
- .expect("MoE tile_counts not initialized"),
- ArrayId::MoeTileOffsets => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_tile_offsets.clone())
- .expect("MoE tile_offsets not initialized"),
- ArrayId::MoeTileMap => {
- self.llm_aux.as_ref().and_then(|aux| aux.moe_tile_map.clone()).expect("MoE tile_map not initialized")
- },
- ArrayId::MoeTotalTiles => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_total_tiles.clone())
- .expect("MoE total_tiles not initialized"),
- ArrayId::MoeDispatchArgs => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_dispatch_args.clone())
- .expect("MoE dispatch_args not initialized"),
- ArrayId::MoeScatterPartials => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_scatter_partials.clone())
- .expect("MoE scatter_partials not initialized"),
- ArrayId::MoeScatterBlockBases => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_scatter_block_bases.clone())
- .expect("MoE scatter_block_bases not initialized"),
- ArrayId::MoeBlockAlloc => self
- .llm_aux
- .as_ref()
- .and_then(|aux| aux.moe_block_alloc.clone())
- .expect("MoE block_alloc not initialized"),
-
- // Classifier-specific arrays
- ArrayId::ClassifierPooling => self.classifier_state().pooling.clone(),
- ArrayId::ClassifierPredictionHeadDense => self.classifier_state().dense.clone(),
- ArrayId::ClassifierPredictionHeadNorm => self.classifier_state().norm.clone(),
- ArrayId::ClassifierPredictionHeadLogits => self.classifier_state().classifier_logits.clone(),
- }
- }
-
- fn shared_buffer_array(
- &self,
- id: ArrayId,
- ) -> Option<Array<B>> {
- let shared = self.shared_buffers.borrow();
- match id {
- ArrayId::RopeCosines(rope_type) => Some(match rope_type {
- RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").cosines.clone(),
- RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").cosines.clone(),
- }),
- ArrayId::RopeSines(rope_type) => Some(match rope_type {
- RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").sines.clone(),
- RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").sines.clone(),
- }),
- _ => None,
- }
- }
-
- pub fn conv_padded_buffer(&self) -> Option<Array<B>> {
- self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())
- }
-
- // ========================================================================
- // Public API Methods (formerly trait methods)
- // ========================================================================
-
- pub fn active_row_count(&self) -> usize {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count,
- ForwardPassMode::Classifier(state) => state.active_row_count,
- }
- }
-
- pub fn set_active_row_count(
- &mut self,
- active_row_count: usize,
- ) {
- match &mut self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count = active_row_count,
- ForwardPassMode::Classifier(state) => state.active_row_count = active_row_count,
- }
- }
-
- /// Start index (within the suffix batch) for which we need logits/sampling.
- pub fn sampling_start(&self) -> usize {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_start,
- ForwardPassMode::Classifier(_) => 0,
- }
- }
-
- /// Number of batch items for which we need logits/sampling.
- pub fn sampling_length(&self) -> usize {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_length,
- ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,
- }
- }
-
- pub fn is_prefilling(&self) -> bool {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.is_prefilling,
- ForwardPassMode::Classifier(_) => true,
- }
- }
-
- pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers<B>>>> {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => Some(&state.cache_layers),
- ForwardPassMode::Classifier(_) => None,
- }
- }
-
- pub fn sampling_output(&self) -> Option<&Array<B>> {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_output.as_ref(),
- ForwardPassMode::Classifier(_) => None,
- }
- }
-
- #[cfg(feature = "tracing")]
- pub fn traces(&self) -> &Rc<RefCell<ActivationTrace<B>>> {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => &state.traces,
- ForwardPassMode::Classifier(state) => &state.traces,
- }
- }
-
- pub fn sampling_method_mut(&mut self) -> Option<&mut Option<SamplingMethod>> {
- match &mut self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => Some(&mut state.sampling_method),
- ForwardPassMode::Classifier(_) => None,
- }
- }
-
- pub fn sampling_method(&self) -> Option<SamplingMethod> {
- match &self.mode {
- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_method,
- ForwardPassMode::Classifier(_) => None,
- }
- }
-
- #[cfg(feature = "tracing")]
- pub fn encode_copy_array(
- &self,
- encoder: &mut Encoder<B>,
- source_array_id: ArrayId,
- destination_array: Array<B>,
- ) {
- let source_array = self.array(source_array_id);
-
- let src_buf_rc = source_array.buffer();
- let dst_buf_rc = destination_array.buffer();
-
- let copy_size_bytes = destination_array.size();
- debug_assert_eq!(destination_array.size(), source_array.size());
-
- encoder.encode_copy(
- src_buf_rc.borrow().deref(),
- 0..copy_size_bytes,
- dst_buf_rc.borrow_mut().deref_mut(),
- 0..copy_size_bytes,
- );
- }
-}
diff --git a/crates/backend-uzu/src/forward_pass/token_inputs.rs b/crates/backend-uzu/src/forward_pass/token_inputs.rs
new file mode 100644
index 000000000..2c07be9f6
--- /dev/null
+++ b/crates/backend-uzu/src/forward_pass/token_inputs.rs
@@ -0,0 +1,174 @@
+use ndarray::ArrayView2;
+
+#[cfg(feature = "tracing")]
+use crate::forward_pass::traces::ActivationTrace;
+use crate::{
+ DataType,
+ array::{Array, ArrayContextExt},
+ backends::common::{Allocation, Backend},
+ encodable_block::DecoderArguments,
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
+};
+
+pub struct TokenInputs<B: Backend> {
+ token_ids: Array<B>,
+ token_subtrie_ranges: Option<Array<B>>,
+ token_positions: Array<B>,
+ token_parents: Array<B>,
+}
+
+impl<B: Backend> TokenInputs<B> {
+ pub fn new_llm(
+ context: &B::Context,
+ model_shape: &ModelShape,
+ token_ids: &[u64],
+ token_subtrie_ranges: Option<&[[u32; 3]]>,
+ token_positions: &[usize],
+ token_ids_array: Option<Array<B>>,
+ token_positions_array: Option<Array<B>>,
+ sampling_start: usize,
+ sampling_length: usize,
+ ) -> Self {
+ let suffix_length = token_ids.len();
+ assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");
+
+ Self {
+ token_ids: token_ids_array.unwrap_or_else(|| Self::init_token_ids(context, token_ids)),
+ token_subtrie_ranges: token_subtrie_ranges
+ .map(|ranges| Self::init_token_subtrie_ranges(context, model_shape, suffix_length, ranges)),
+ token_positions: token_positions_array
+ .unwrap_or_else(|| Self::init_token_positions(context, token_positions)),
+ token_parents: Self::init_token_parents(context, token_positions, sampling_start, sampling_length),
+ }
+ }
+
+ pub fn new_classifier(
+ context: &B::Context,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ ) -> Self {
+ let suffix_length = token_ids.len();
+ assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");
+
+ Self {
+ token_ids: Self::init_token_ids(context, token_ids),
+ token_subtrie_ranges: None,
+ token_positions: Self::init_token_positions(context, token_positions),
+ token_parents: Self::init_token_parents(context, token_positions, 0, 0),
+ }
+ }
+
+ pub fn token_ids(&self) -> &Allocation<B> {
+ self.token_ids.allocation()
+ }
+
+ pub fn token_positions(&self) -> &Allocation<B> {
+ self.token_positions.allocation()
+ }
+
+ pub fn token_parents(&self) -> &Allocation<B> {
+ self.token_parents.allocation()
+ }
+
+ pub fn token_subtrie_ranges(&self) -> Option<&Allocation<B>> {
+ self.token_subtrie_ranges.as_ref().map(Array::allocation)
+ }
+
+ pub fn decoder_arguments<'a>(
+ &'a self,
+ model_shape: &ModelShape,
+ shared_buffers: &'a SharedBuffers<B>,
+ cache_layers: Option<&'a mut CacheLayers<B>>,
+ batch_dim: usize,
+ sampling_start: usize,
+ sampling_length: usize,
+ #[cfg(feature = "tracing")] trace: Option<&'a mut ActivationTrace<B>>,
+ ) -> DecoderArguments<'a, B> {
+ DecoderArguments {
+ token_positions: self.token_positions(),
+ token_parents: self.token_parents(),
+ token_subtrie_ranges: self.token_subtrie_ranges(),
+ shared_buffers,
+ cache_layers,
+ batch_dim,
+ sampling_start,
+ sampling_length,
+ rope_max_sequence_length: model_shape.context_length(),
+ rope_dim: model_shape.rope_dim(),
+ #[cfg(feature = "tracing")]
+ trace,
+ }
+ }
+
+ fn init_token_ids(
+ context: &B::Context,
+ token_ids: &[u64],
+ ) -> Array<B> {
+ context.create_array_from(&[token_ids.len()], token_ids)
+ }
+
+ fn init_token_positions(
+ context: &B::Context,
+ token_positions: &[usize],
+ ) -> Array<B> {
+ let positions_i32: Box<[i32]> = token_positions.iter().map(|position| *position as i32).collect();
+ context.create_array_from(&[token_positions.len()], positions_i32.as_ref())
+ }
+
+ fn init_token_parents(
+ context: &B::Context,
+ token_positions: &[usize],
+ sampling_start: usize,
+ sampling_length: usize,
+ ) -> Array<B> {
+ let suffix_length = token_positions.len();
+ let mut parents = vec![-1i32; suffix_length];
+
+ if sampling_length > 0 {
+ let root_pos = token_positions.get(sampling_start).copied().unwrap_or(0);
+ let mut stack: Vec<usize> = Vec::new();
+
+ for local_idx in 0..sampling_length {
+ let abs_idx = sampling_start + local_idx;
+ let Some(&pos) = token_positions.get(abs_idx) else {
+ break;
+ };
+ let depth = pos.saturating_sub(root_pos);
+ let parent_local = if depth == 0 {
+ -1
+ } else if let Some(&parent) = stack.get(depth - 1) {
+ parent as i32
+ } else {
+ debug_assert!(false, "invalid trie depth ordering: depth={depth}, stack_len={}", stack.len());
+ -1
+ };
+ parents[abs_idx] = parent_local;
+
+ if stack.len() <= depth {
+ stack.resize(depth + 1, 0);
+ }
+ stack[depth] = local_idx;
+ stack.truncate(depth + 1);
+ }
+ }
+
+ context.create_array_from(&[suffix_length], &parents)
+ }
+
+ fn init_token_subtrie_ranges(
+ context: &B::Context,
+ model_shape: &ModelShape,
+ suffix_length: usize,
+ token_subtrie_ranges: &[[u32; 3]],
+ ) -> Array<B> {
+ let shape = model_shape.subtrie_ranges_shape(suffix_length);
+ let mut array = context.create_array_zeros(&shape, DataType::U32);
+ let source = ArrayView2::from_shape(
+ (token_subtrie_ranges.len(), 3),
+ bytemuck::cast_slice::<_, u32>(token_subtrie_ranges),
+ )
+ .expect("Invalid token subtrie range shape");
+ array.copy_from_view(source);
+ array
+ }
+}
diff --git a/crates/backend-uzu/src/forward_pass/traces.rs b/crates/backend-uzu/src/forward_pass/traces.rs
index 37b4cb589..820073ff1 100644
--- a/crates/backend-uzu/src/forward_pass/traces.rs
+++ b/crates/backend-uzu/src/forward_pass/traces.rs
@@ -1,29 +1,15 @@
-use std::{cell::RefCell, rc::Rc};
-
use crate::{
- DataType,
array::{Array, ArrayContextExt},
backends::common::Backend,
forward_pass::model_shape::ModelShape,
};
-fn create_trace_array<B: Backend>(
- context: &B::Context,
- shape: &[usize],
- data_type: DataType,
- label: &str,
-) -> Array<B> {
- context.create_array_uninitialized(shape, data_type, label)
-}
-
fn create_layer_results<B: Backend>(
context: &B::Context,
model_shape: &ModelShape,
suffix_length: usize,
-) -> Vec<Rc<RefCell<LayerActivationTrace<B>>>> {
- (0..model_shape.num_layers)
- .map(|_| Rc::new(RefCell::new(LayerActivationTrace::new(context, model_shape, suffix_length))))
- .collect()
+) -> Box<[LayerActivationTrace<B>]> {
+ (0..model_shape.num_layers).map(|_| LayerActivationTrace::new(context, model_shape, suffix_length)).collect()
}
pub struct LayerActivationTrace<B: Backend> {
@@ -46,25 +32,25 @@ impl<B: Backend> LayerActivationTrace<B> {
) -> Self {
let main_shape = model_shape.main_shape(suffix_length);
let activation_data_type = model_shape.activation_data_type();
- let main = |label| create_trace_array(context, &main_shape, activation_data_type, label);
+ let main = || context.create_array_uninitialized(&main_shape, activation_data_type);
Self {
- inputs: main("layer_activation_trace_inputs"),
- pre_attention_norm: main("layer_activation_trace_pre_attention_norm"),
- attention: main("layer_activation_trace_attention"),
- post_attention_norm: main("layer_activation_trace_post_attention_norm"),
- mlp_inputs: main("layer_activation_trace_mlp_inputs"),
- pre_mlp_norm: main("layer_activation_trace_pre_mlp_norm"),
- mlp: main("layer_activation_trace_mlp"),
- post_mlp_norm: main("layer_activation_trace_post_mlp_norm"),
- outputs: main("layer_activation_trace_outputs"),
+ inputs: main(),
+ pre_attention_norm: main(),
+ attention: main(),
+ post_attention_norm: main(),
+ mlp_inputs: main(),
+ pre_mlp_norm: main(),
+ mlp: main(),
+ post_mlp_norm: main(),
+ outputs: main(),
}
}
}
pub struct ActivationTrace<B: Backend> {
pub embedding_norm: Option<Array<B>>,
- pub layer_results: Vec<Rc<RefCell<LayerActivationTrace<B>>>>,
+ pub layer_results: Box<[LayerActivationTrace<B>]>,
pub output_norm: Array<B>,
pub output_pooling: Option<Array<B>>,
pub logits: Array<B>,
@@ -83,14 +69,9 @@ impl<B: Backend> ActivationTrace<B> {
Self {
embedding_norm: None,
layer_results,
- output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),
+ output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),
output_pooling: None,
- logits: create_trace_array(
- context,
- &model_shape.logits_shape(suffix_length),
- activation_data_type,
- "activation_trace_logits",
- ),
+ logits: context.create_array_uninitialized(&model_shape.logits_shape(suffix_length), activation_data_type),
}
}
@@ -106,29 +87,19 @@ impl<B: Backend> ActivationTrace<B> {
let model_dim = model_shape.main_shape(1)[1];
Self {
- embedding_norm: Some(create_trace_array(
- context,
- &main_shape,
- activation_data_type,
- "activation_trace_embedding_norm",
- )),
+ embedding_norm: Some(context.create_array_uninitialized(&main_shape, activation_data_type)),
layer_results,
- output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),
- output_pooling: Some(create_trace_array(
- context,
- &[1, model_dim],
- activation_data_type,
- "activation_trace_output_pooling",
- )),
- logits: create_trace_array(context, &[1, num_labels], activation_data_type, "activation_trace_logits"),
+ output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),
+ output_pooling: Some(context.create_array_uninitialized(&[1, model_dim], activation_data_type)),
+ logits: context.create_array_uninitialized(&[1, num_labels], activation_data_type),
}
}
- pub fn embedding_norm(&self) -> &Array<B> {
- self.embedding_norm.as_ref().expect("embedding_norm is only available for classifier traces")
+ pub fn embedding_norm_mut(&mut self) -> &mut Array<B> {
+ self.embedding_norm.as_mut().expect("embedding_norm is only available for classifier traces")
}
- pub fn output_pooling(&self) -> &Array<B> {
- self.output_pooling.as_ref().expect("output_pooling is only available for classifier traces")
+ pub fn output_pooling_mut(&mut self) -> &mut Array<B> {
+ self.output_pooling.as_mut().expect("output_pooling is only available for classifier traces")
}
}
diff --git a/crates/backend-uzu/src/language_model/grammar/mod.rs b/crates/backend-uzu/src/language_model/grammar/mod.rs
index 43e9e01c3..bf464d854 100644
--- a/crates/backend-uzu/src/language_model/grammar/mod.rs
+++ b/crates/backend-uzu/src/language_model/grammar/mod.rs
@@ -20,10 +20,11 @@ pub trait CompiledGrammar {
fn is_terminated(&self) -> bool;
}
+#[cfg_attr(not(grammar_xgrammar), allow(unused_variables))]
pub fn create_compiled_grammar(
- #[allow(unused)] config: &GrammarConfig,
- #[allow(unused)] tokenizer: &Tokenizer,
- #[allow(unused)] stop_token_ids: Option<&[i32]>,
+ config: &GrammarConfig,
+ tokenizer: &Tokenizer,
+ stop_token_ids: Option<&[i32]>,
) -> Result<Box<dyn CompiledGrammar>, Error> {
#[cfg(grammar_xgrammar)]
{
@@ -38,5 +39,7 @@ pub fn create_compiled_grammar(
}
#[cfg(not(grammar_xgrammar))]
- Err(Error::GrammarNoBackendAvailable)
+ {
+ Err(Error::GrammarNoBackendAvailable)
+ }
}
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index a0b48f61c..79c13d59b 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -1,10 +1,4 @@
-use std::{
- any::Any,
- iter::repeat_n,
- ops::{Deref, DerefMut, Range},
- path::Path,
- time::Instant,
-};
+use std::{any::Any, iter::repeat_n, ops::Range, path::Path, time::Instant};
use itertools::{Either, Itertools, izip};
@@ -15,17 +9,14 @@ use super::{
rng::PRng,
};
use crate::{
+ array::{Array, ArrayContextExt},
backends::common::{
- Backend, CommandBuffer, Context, DenseBuffer, Encoder, Executable,
- kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},
+ Allocation, AsBufferRangeRef, Backend, CommandBuffer, Context, DenseBuffer, Encoder, Pending,
+ kernel::TokenCopySampledKernel,
},
config::ModelMetadata,
- encodable_block::EncodingParameters,
- forward_pass::{
- cache_layers::{CacheLayer, CacheLayersSlice},
- kv_cache_layer::INVALID_POSITION,
- state::ForwardPassState,
- },
+ encodable_block::{DecoderDecodeInput, EncodingParameters, SamplingArguments, SamplingInputs},
+ forward_pass::{cache_layers::CacheLayersSlice, kv_cache_layer::INVALID_POSITION, token_inputs::TokenInputs},
language_model::grammar::CompiledGrammar,
session::{
config::DecodingConfig,
@@ -44,24 +35,22 @@ struct Task<'a> {
token_positions: &'a [usize],
token_bitmask: Option<&'a [u32]>,
token_seeds: &'a [u64],
- expected_number_of_new_tokens: usize,
active_row_count: usize,
sampling_start: usize,
sampling_length: usize,
is_prefilling: bool,
}
-#[derive(Debug, Clone, PartialEq)]
-struct TaskEncodingKey {
- context_len: usize,
- batch_size: usize,
- expected_number_of_new_tokens: usize,
- active_row_count: usize,
- sampling_method: SamplingMethod,
- sampling_start: usize,
- sampling_len: usize,
- has_bitmask: bool,
- is_prefilling: bool,
+struct ForwardPassResources<B: Backend> {
+ token_inputs: TokenInputs<B>,
+ logits: Option<Allocation<B>>,
+ sampling_output: Option<Allocation<B>>,
+ sampling_inputs: Option<SamplingInputs<B>>,
+}
+
+struct InFlightForwardPass<B: Backend> {
+ resources: ForwardPassResources<B>,
+ pending: Pending<B>,
}
pub struct LanguageModelGenerator<B: Backend> {
@@ -69,7 +58,8 @@ pub struct LanguageModelGenerator<B: Backend> {
pub tokens: Vec<u64>,
pub context: LanguageModelGeneratorContext<B>,
- pre_encoded_task: Option<(TaskEncodingKey, Executable<B>)>,
+ async_in_flight: Vec<Option<InFlightForwardPass<B>>>,
+ async_token_ids: Option<Array<B>>,
registered_prefix_len: usize,
gpu_capture: GpuCaptureManager<B>,
}
@@ -100,6 +90,10 @@ pub trait LanguageModelGeneratorTrait {
sampling_method: SamplingMethod,
on_complete: Box<dyn FnOnce(u64) + Send>,
) -> Result<(), Error>;
+ fn finish_async(
+ &mut self,
+ pass_idx: usize,
+ );
fn clear_cache(&mut self);
fn reset_state(&mut self);
@@ -187,7 +181,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let token_seeds = repeat_n(0, tokens_length - 1).chain(flat_trie.token_seeds()).chunks(prefill_step_size);
- let mut last_state: Option<ForwardPassState<B>> = None;
+ let mut last_sampling_output: Option<Allocation<B>> = None;
+ let mut last_sampling_length = 0;
let mut run_times: Vec<f64> = Vec::new();
// Process each prefill step and update the KV cache.
@@ -269,7 +264,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");
}
- let _ = last_state.take();
+ let _ = last_sampling_output.take();
let task = Task {
token_ids: &step_token_ids,
@@ -277,14 +272,13 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
token_positions: &step_token_positions,
token_bitmask: step_token_bitmask.as_deref(),
token_seeds: &step_token_seeds,
- expected_number_of_new_tokens: step_token_ids.len(),
active_row_count,
sampling_start,
sampling_length,
is_prefilling: !should_sample_after_step,
};
- let (state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;
+ let (sampling_output, run_time) = self.run_model(task, sampling_method)?;
if should_capture {
self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;
@@ -306,11 +300,12 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.registered_prefix_len = prefix_offset + tokens_start_index + tokens_processed_this_step;
}
- last_state = Some(state);
+ last_sampling_length = sampling_length;
+ last_sampling_output = sampling_output;
run_times.push(run_time);
}
- let mut final_state = last_state.ok_or(Error::PrefillFailed)?;
+ let final_sampling_output = last_sampling_output;
if !sample_suffix {
self.sync_prefix();
return Ok(PrefillResult {
@@ -318,7 +313,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
forwardpass_durations: run_times,
});
}
- let sampled_tokens = self.read_sampling_output(&mut final_state)?;
+ let sampled_tokens = self.read_sampling_output(
+ final_sampling_output.as_ref().expect("sampling output must exist"),
+ last_sampling_length,
+ )?;
let last_suffix_start = prefill_step_size * (prefill_steps - 1);
let suffix_root_index = (tokens_length - last_suffix_start) - 1;
@@ -403,16 +401,16 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
token_positions: &token_positions,
token_bitmask: token_bitmask.as_deref(),
token_seeds: &token_seeds,
- expected_number_of_new_tokens: 1,
active_row_count,
sampling_start: 0,
sampling_length: active_row_count,
is_prefilling: false,
};
- let (mut state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;
-
- let sampled_tokens = self.read_sampling_output(&mut state)?;
+ let sampling_length = task.sampling_length;
+ let (sampling_output, run_time) = self.run_model(task, sampling_method)?;
+ let sampled_tokens =
+ self.read_sampling_output(sampling_output.as_ref().expect("sampling output must exist"), sampling_length)?;
let (accepted_tokens, accepted_token_indices) =
flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());
@@ -443,6 +441,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);
self.context.async_buffers.prepare_seeds(&self.context.seed, prefill_count, tokens_to_generate);
self.context.async_buffers.reset_counter();
+ self.async_in_flight = (0..self.context.async_buffers.batch_size).map(|_| None).collect();
+ self.async_token_ids = None;
}
/// Submits a single async forward pass.
@@ -465,16 +465,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let batch_size = self.context.async_buffers.batch_size;
let slot = pass_idx % batch_size;
- let results_buffer = self.context.async_buffers.results.clone();
- let async_positions_buffer = self.context.async_buffers.positions.clone();
- let async_seeds_buffer = self.context.async_buffers.seeds.clone();
-
let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
-
- let token_position = unsafe {
- let ptr = async_positions_buffer.borrow().cpu_ptr().as_ptr() as *const u32;
- *ptr.add(pass_idx) as usize
- };
+ let token_position = self.context.async_buffers.positions.as_slice::<i32>()[pass_idx] as usize;
+ let token_seed = self.context.async_buffers.seeds.as_slice::<u64>()[pass_idx];
let task = Task {
token_ids: &[last_token],
@@ -482,17 +475,18 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
token_positions: &[token_position],
token_bitmask: None,
token_seeds: &[0], // Ignored, using async buffer
- expected_number_of_new_tokens: 1,
active_row_count: 1,
sampling_start: 0,
sampling_length: 1,
is_prefilling: false,
};
- let async_positions = Some((async_positions_buffer.clone(), pass_idx));
- let async_seeds = Some((async_seeds_buffer.clone(), pass_idx));
-
- let skip_token_ids_copy = pass_idx > 0;
+ let token_ids_array = if pass_idx > 0 {
+ Some(self.async_token_ids.take().expect("previous async pass must provide token_ids array"))
+ } else {
+ None
+ };
+ let sampling_inputs = SamplingInputs::from_slices(self.context.context.as_ref(), &[token_seed], None, None);
let is_first_decode = !is_continuation;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
@@ -500,65 +494,47 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let _ = self.gpu_capture.start_capture(&self.context.context, "decode");
}
- let mut state = ForwardPassState::new_llm(
- self.context.context.clone(),
- &self.context.decoder_config,
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.cache_layers.clone(),
- self.context.shared_buffers.clone(),
- &task.token_ids,
- task.token_subtrie_ranges,
- &task.token_positions,
- task.token_bitmask,
- &task.token_seeds,
- task.active_row_count,
- /*sampling_start=*/ 0,
- /*sampling_length=*/ task.active_row_count,
- task.is_prefilling,
- skip_token_ids_copy,
- async_positions,
- async_seeds,
- );
- if let Some(sm) = state.sampling_method_mut() {
- *sm = Some(sampling_method);
- }
-
+ let batch_dim = task.active_row_count;
+ let sampling_start = task.sampling_start;
+ let sampling_length = task.sampling_length;
+ let token_inputs = self.build_token_inputs(task, token_ids_array, None);
let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
.map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
-
- // Wait on previous pass if this is a continuation
if is_continuation {
encoder.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
}
-
- self.context
- .executables
- .encode(&mut state, &EncodingParameters::new(), &mut encoder)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- // Encode sampling
- self.context.gpu_sampler.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ let resources = Self::encode_forward_pass_on(
+ &self.context,
+ &mut encoder,
+ token_inputs,
+ batch_dim,
+ sampling_start,
+ sampling_length,
+ false,
+ Some(sampling_method),
+ &EncodingParameters::new(),
+ Some(sampling_inputs),
+ )?;
// Copy sampled token: sampling_output → token_ids (for next pass)
- // and sampling_output → results[slot] (for callback)
- let sampling_output = state.sampling_output().expect("sampling_output must exist after sampling encode");
- let sampling_output_buf_rc = sampling_output.buffer();
- let sampling_output_buf_borrow = sampling_output_buf_rc.borrow();
- let token_ids_buf_rc = self.context.scratch_buffers.token_ids.buffer();
- let mut token_ids_buf_borrow = token_ids_buf_rc.borrow_mut();
-
+ let token_ids_shape = [1];
+ let token_ids_data_type = crate::DataType::U64;
+ let mut async_token_ids_allocation =
+ self.context.context.create_array_uninitialized(&token_ids_shape, token_ids_data_type).into_allocation();
+ let async_token_ids_buffer_range = async_token_ids_allocation.as_buffer_range_ref();
+ let async_token_ids_range = async_token_ids_buffer_range.range();
+ let async_token_ptr = SendPtr(unsafe {
+ (async_token_ids_buffer_range.buffer().cpu_ptr().as_ptr() as *const u64)
+ .add(async_token_ids_range.start / std::mem::size_of::<u64>())
+ });
self.context.token_copy_sampled.encode(
- sampling_output_buf_borrow.deref(),
- token_ids_buf_borrow.deref_mut(),
- &mut encoder,
- );
- let results_offset = slot * std::mem::size_of::<u32>();
- self.context.token_copy_results.encode(
- sampling_output_buf_borrow.deref(),
- (results_buffer.borrow_mut().deref_mut(), results_offset),
+ resources.sampling_output.as_ref().expect("Sampling output must exist"),
+ &mut async_token_ids_allocation,
&mut encoder,
);
+ self.async_token_ids = Some(unsafe {
+ Array::from_allocation(async_token_ids_allocation, 0, &token_ids_shape, token_ids_data_type)
+ });
// Scatter + register for all transformer layers
self.context.cache_layers.borrow_mut().update_after_acceptance(
@@ -575,11 +551,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
- let results_buffer_ptr = SendPtr(results_buffer.borrow().cpu_ptr().as_ptr() as *const u32);
-
let handler = move |result: Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>| {
result.expect("async decoding forward pass completed with error");
- let token = { unsafe { *results_buffer_ptr.as_ptr().add(slot) as u64 } };
+ let token = unsafe { *async_token_ptr.as_ptr() };
on_complete(token);
};
@@ -588,22 +562,43 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let pending = encoder.end_encoding().submit();
if should_capture {
- pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ let completed = pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ drop(resources);
+ drop(completed);
self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
+ } else {
+ assert!(self.async_in_flight[slot].is_none(), "async slot {slot} still holds an in-flight state");
+ self.async_in_flight[slot] = Some(InFlightForwardPass {
+ resources,
+ pending,
+ });
}
Ok(())
}
- fn clear_cache(&mut self) {
- self.pre_encoded_task = None;
+ fn finish_async(
+ &mut self,
+ pass_idx: usize,
+ ) {
+ let slot = pass_idx % self.context.async_buffers.batch_size;
+ if let Some(in_flight) = self.async_in_flight[slot].take() {
+ let InFlightForwardPass {
+ resources,
+ pending,
+ } = in_flight;
+ drop((resources, pending));
+ }
}
+ fn clear_cache(&mut self) {}
+
fn reset_state(&mut self) {
- self.context.cache_layers.borrow_mut().clear();
+ self.context.cache_layers.borrow_mut().clear(self.context.context.as_ref());
self.tokens.clear();
self.registered_prefix_len = 0;
- self.pre_encoded_task = None;
+ self.async_in_flight.clear();
+ self.async_token_ids = None;
self.gpu_capture.reset();
let seed = self.decoding_config.sampling_seed.resolve();
@@ -659,29 +654,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
context: &dyn Any,
) {
let ctx = context.downcast_ref::<LlmContext<B>>().unwrap();
- let mut llm_state = self.context.cache_layers.borrow_mut();
- for (_layer_idx, (ctx_layer, gen_layer)) in
- ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()
- {
- match (ctx_layer, gen_layer) {
- (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {
- let copy_rows = src.prefix_segment_length();
- if copy_rows > 0 {
- dst.keys.copy_slice(&src.keys, 0, 0..copy_rows, 0);
- dst.values.copy_slice(&src.values, 0, 0..copy_rows, 0);
- }
- dst.state = src.state.clone();
- },
- (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {
- dst.conv_state.copy_from_array(&src.conv_state);
- dst.ssm_state.copy_from_array(&src.ssm_state);
- },
- _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),
- }
- }
- drop(llm_state);
-
+ self.context.cache_layers.borrow_mut().copy_from(&ctx.cache_layers, self.context.context.as_ref());
self.tokens = ctx.tokens.clone();
+ self.registered_prefix_len = self.tokens.len().saturating_sub(1);
}
}
@@ -699,7 +674,8 @@ impl<B: Backend> LanguageModelGenerator<B> {
decoding_config,
tokens: Vec::new(),
context,
- pre_encoded_task: None,
+ async_in_flight: Vec::new(),
+ async_token_ids: None,
registered_prefix_len: 0,
gpu_capture,
})
@@ -708,80 +684,52 @@ impl<B: Backend> LanguageModelGenerator<B> {
fn run_model(
&mut self,
task: Task,
- allow_pre_encode: bool,
sampling_method: SamplingMethod,
- ) -> Result<(ForwardPassState<B>, f64), Error> {
+ ) -> Result<(Option<Allocation<B>>, f64), Error> {
let run_start = Instant::now();
-
- let mut state = ForwardPassState::new_llm(
- self.context.context.clone(),
- &self.context.decoder_config,
- &self.context.model_shape,
- &self.context.scratch_buffers,
- self.context.cache_layers.clone(),
- self.context.shared_buffers.clone(),
- task.token_ids,
- task.token_subtrie_ranges,
- task.token_positions,
- task.token_bitmask,
- task.token_seeds,
- task.active_row_count,
- task.sampling_start,
- task.sampling_length,
- task.is_prefilling,
- false,
- None,
- None,
- );
-
- if let Some(method) = state.sampling_method_mut() {
- *method = Some(sampling_method);
- }
-
- let encoding_key = TaskEncodingKey {
- context_len: self.tokens.len(),
- batch_size: task.token_ids.len(),
- expected_number_of_new_tokens: task.expected_number_of_new_tokens,
- active_row_count: task.active_row_count,
- sampling_method,
- sampling_start: task.sampling_start,
- sampling_len: task.sampling_length,
- has_bitmask: task.token_bitmask.is_some(),
- is_prefilling: task.is_prefilling,
- };
+ let sample = !task.is_prefilling;
+ let is_prefilling = task.is_prefilling;
+ let batch_dim = task.active_row_count;
+ let sampling_start = task.sampling_start;
+ let sampling_length = task.sampling_length;
+ let sampling_inputs = sample.then(|| {
+ self.create_sampling_inputs(task.sampling_start, task.sampling_length, task.token_seeds, task.token_bitmask)
+ });
let is_first_decode = task.token_ids.len() == 1;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
+ let token_inputs = self.build_token_inputs(task, None, None);
if should_capture {
self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
- self.pre_encoded_task = None;
}
- let sample = !task.is_prefilling;
-
- let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()
- && pre_encoded_key == encoding_key
- {
- pre_encoded_executable
- } else {
- self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?
- };
-
- let pending = executable.submit();
-
- if allow_pre_encode {
- let mut next_encoding_key = encoding_key;
-
- next_encoding_key.context_len += 1;
-
- let next_executable =
- self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;
+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
+ let resources = Self::encode_forward_pass_on(
+ &self.context,
+ &mut encoder,
+ token_inputs,
+ batch_dim,
+ sampling_start,
+ sampling_length,
+ is_prefilling,
+ sample.then_some(sampling_method),
+ &EncodingParameters::new(),
+ sampling_inputs,
+ )?;
- self.pre_encoded_task = Some((next_encoding_key, next_executable));
- }
+ let pending = encoder.end_encoding().submit();
- pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ let completed = pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ let ForwardPassResources {
+ token_inputs,
+ logits,
+ sampling_output,
+ sampling_inputs,
+ } = resources;
+ drop((token_inputs, logits, sampling_inputs));
+ drop(completed);
let run_time = run_start.elapsed().as_secs_f64();
@@ -789,49 +737,140 @@ impl<B: Backend> LanguageModelGenerator<B> {
self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
}
- Ok((state, run_time))
+ Ok((sampling_output, run_time))
}
- fn encode_forward_pass(
- &self,
- state: &mut ForwardPassState<B>,
+ fn encode_forward_pass_on(
+ context: &LanguageModelGeneratorContext<B>,
+ encoder: &mut Encoder<B>,
+ token_inputs: TokenInputs<B>,
+ batch_dim: usize,
+ sampling_start: usize,
+ sampling_length: usize,
+ is_prefilling: bool,
+ sampling_method: Option<SamplingMethod>,
parameters: &EncodingParameters,
- sample: bool,
- ) -> Result<Executable<B>, Error> {
- let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
-
- self.context
- .executables
- .encode(state, parameters, &mut encoder)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- if sample {
- self.context.gpu_sampler.encode(state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ sampling_inputs: Option<SamplingInputs<B>>,
+ ) -> Result<ForwardPassResources<B>, Error> {
+ let mut sampling_output = sampling_method.map(|_| {
+ context.context.create_array_uninitialized(&[sampling_length], crate::DataType::U32).into_allocation()
+ });
+ let mut logits = None;
+ if is_prefilling {
+ let mut cache_layers = context.cache_layers.borrow_mut();
+ let decoder_arguments = token_inputs.decoder_arguments(
+ &context.model_shape,
+ context.shared_buffers.as_ref(),
+ Some(&mut *cache_layers),
+ batch_dim,
+ sampling_start,
+ sampling_length,
+ #[cfg(feature = "tracing")]
+ None,
+ );
+ context
+ .executables
+ .encode_prefill(decoder_arguments, token_inputs.token_ids(), parameters, encoder)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ } else {
+ let mut cache_layers = context.cache_layers.borrow_mut();
+ let decoder_arguments = token_inputs.decoder_arguments(
+ &context.model_shape,
+ context.shared_buffers.as_ref(),
+ Some(&mut *cache_layers),
+ batch_dim,
+ sampling_start,
+ sampling_length,
+ #[cfg(feature = "tracing")]
+ None,
+ );
+ let mut retained_logits = context
+ .executables
+ .encode_decode(
+ decoder_arguments,
+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),
+ None,
+ parameters,
+ encoder,
+ )
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+
+ let sampling_inputs = sampling_inputs.as_ref().expect("Sampling requires sampling inputs");
+ let sampling_result = context.gpu_sampler.encode(
+ SamplingArguments {
+ logits: &mut retained_logits,
+ seeds: sampling_inputs.seeds.allocation(),
+ bitmask: sampling_inputs.bitmask.as_ref().map(Array::allocation),
+ output: sampling_output.as_mut().expect("Sampling requires output allocation"),
+ sampling_method: sampling_method.expect("Sampling requires method"),
+ batch_size: sampling_length,
+ vocab_size: context.model_config.model_config.vocab_size,
+ },
+ encoder,
+ );
+ sampling_result.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ logits = Some(retained_logits);
}
- let executable = encoder.end_encoding();
+ Ok(ForwardPassResources {
+ token_inputs,
+ logits,
+ sampling_output,
+ sampling_inputs,
+ })
+ }
+
+ fn build_token_inputs(
+ &self,
+ task: Task<'_>,
+ token_ids_array: Option<Array<B>>,
+ token_positions_array: Option<Array<B>>,
+ ) -> TokenInputs<B> {
+ TokenInputs::new_llm(
+ self.context.context.as_ref(),
+ &self.context.model_shape,
+ task.token_ids,
+ task.token_subtrie_ranges,
+ task.token_positions,
+ token_ids_array,
+ token_positions_array,
+ task.sampling_start,
+ task.sampling_length,
+ )
+ }
- Ok(executable)
+ fn create_sampling_inputs(
+ &self,
+ sampling_start: usize,
+ sampling_length: usize,
+ token_seeds: &[u64],
+ token_bitmask: Option<&[u32]>,
+ ) -> SamplingInputs<B> {
+ let seeds = &token_seeds[sampling_start..sampling_start + sampling_length];
+ let bitmask_row_len = token_bitmask.map(|_| self.context.model_shape.bitmask_shape(1)[1]);
+ let bitmask = token_bitmask.map(|mask| {
+ let row_len = bitmask_row_len.expect("bitmask row length must exist");
+ let start = sampling_start * row_len;
+ let end = start + sampling_length * row_len;
+ &mask[start..end]
+ });
+ SamplingInputs::from_slices(self.context.context.as_ref(), seeds, bitmask, bitmask_row_len)
}
fn read_sampling_output(
- &mut self,
- state: &mut ForwardPassState<B>,
+ &self,
+ sampling_output: &Allocation<B>,
+ batch_size: usize,
) -> Result<Vec<u64>, Error> {
- let sampling_output = state
- .sampling_output()
- .expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");
-
- let output_view = sampling_output.as_view::<u32>();
- let batch_size = state.sampling_length();
-
- let mut result = Vec::with_capacity(batch_size);
- for i in 0..batch_size {
- result.push(output_view[[i]] as u64);
- }
-
- Ok(result)
+ let buffer_range = sampling_output.as_buffer_range_ref();
+ let range = buffer_range.range();
+ let values = unsafe {
+ std::slice::from_raw_parts(
+ (buffer_range.buffer().cpu_ptr().as_ptr() as *const u32).add(range.start / std::mem::size_of::<u32>()),
+ batch_size,
+ )
+ };
+ Ok(values.iter().map(|value| *value as u64).collect())
}
fn update_cache_layers(
@@ -861,14 +900,6 @@ impl<B: Backend> LanguageModelGenerator<B> {
Ok(())
}
- fn allow_pre_encode(&self) -> bool {
- let debug_active = self.context.context.debug_active();
-
- let result = self.decoding_config.allow_pre_encode && !debug_active;
-
- result
- }
-
fn sync_prefix(&mut self) {
if self.tokens.is_empty() {
return;
diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
index 12476d069..a32b45032 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
@@ -7,15 +7,14 @@ use std::{
use crate::{
DataType,
+ array::{Array, ArrayContextExt},
backends::common::{
- Backend, Context, DenseBuffer, Kernels,
- kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},
+ Backend, Context, Kernels,
+ kernel::{TokenCopySampledKernel, kv_cache_update::KVCacheUpdate},
},
- config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
+ config::{LanguageModelConfig, ModelMetadata},
encodable_block::{Decoder, Sampling},
- forward_pass::{
- cache_layers::CacheLayers, model_shape::ModelShape, scratch_buffers::ScratchBuffers, state::SharedBuffers,
- },
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
language_model::rng::PRng,
parameters::ParameterLoader,
session::{
@@ -25,18 +24,15 @@ use crate::{
},
};
-/// Pre-allocated buffers for async generation pipeline.
+/// Pre-allocated state for async generation pipeline.
/// Indexed by pass_idx to avoid race conditions between GPU passes.
pub struct AsyncBuffers<B: Backend> {
- /// Positions buffer: [max_tokens] i32
+ /// Positions array: [max_tokens] i32
/// Pre-populated with [prefill_count, prefill_count+1, ...]
- pub positions: Rc<RefCell<B::DenseBuffer>>,
- /// Seeds buffer: [max_tokens] u64
+ pub positions: Array<B>,
+ /// Seeds array: [max_tokens] u64
/// Pre-populated with deterministic seed sequence
- pub seeds: Rc<RefCell<B::DenseBuffer>>,
- /// Results buffer: [batch_size] u32
- /// Each pass writes its sampled token to results[pass_idx % batch_size]
- pub results: Rc<RefCell<B::DenseBuffer>>,
+ pub seeds: Array<B>,
/// Event for GPU-side synchronization between passes
pub event: B::Event,
/// Current event counter (pass N waits on N, signals N+1)
@@ -53,18 +49,13 @@ impl<B: Backend> AsyncBuffers<B> {
max_tokens: usize,
batch_size: usize,
) -> Self {
- let positions =
- context.create_buffer(max_tokens * std::mem::size_of::<i32>()).expect("Failed to create positions buffer");
- let seeds =
- context.create_buffer(max_tokens * std::mem::size_of::<u64>()).expect("Failed to create seeds buffer");
- let results =
- context.create_buffer(batch_size * std::mem::size_of::<u32>()).expect("Failed to create results buffer");
+ let positions = context.create_array_uninitialized(&[max_tokens], DataType::I32);
+ let seeds = context.create_array_uninitialized(&[max_tokens], DataType::U64);
let event = context.create_event().expect("Failed to create event");
Self {
- positions: Rc::new(RefCell::new(positions)),
- seeds: Rc::new(RefCell::new(seeds)),
- results: Rc::new(RefCell::new(results)),
+ positions,
+ seeds,
event,
counter: Cell::new(0),
prefill_count: Cell::new(0),
@@ -75,32 +66,28 @@ impl<B: Backend> AsyncBuffers<B> {
/// Prepare positions buffer: [prefill_count-1, prefill_count, ...]
/// Uses prefill_count-1 to match sync path which uses self.tokens.len()-1
pub fn prepare_positions(
- &self,
+ &mut self,
prefill_count: usize,
tokens_to_generate: usize,
) {
self.prefill_count.set(prefill_count);
let base_position = prefill_count.saturating_sub(1);
- let ptr = self.positions.borrow().cpu_ptr().as_ptr() as *mut i32;
- for i in 0..tokens_to_generate {
- unsafe {
- *ptr.add(i) = (base_position + i) as i32;
- }
+ let positions = self.positions.as_slice_mut::<i32>();
+ for (index, position) in positions.iter_mut().take(tokens_to_generate).enumerate() {
+ *position = (base_position + index) as i32;
}
}
/// Prepare seeds buffer with deterministic sequence
pub fn prepare_seeds(
- &self,
+ &mut self,
seed: &PRng,
prefix_len: usize,
tokens_to_generate: usize,
) {
- let ptr = self.seeds.borrow().cpu_ptr().as_ptr() as *mut u64;
- for i in 0..tokens_to_generate {
- unsafe {
- *ptr.add(i) = seed.derive((prefix_len + i - 1) as u64);
- }
+ let seeds = self.seeds.as_slice_mut::<u64>();
+ for (index, value) in seeds.iter_mut().take(tokens_to_generate).enumerate() {
+ *value = seed.derive((prefix_len + index - 1) as u64);
}
}
@@ -114,11 +101,9 @@ pub struct LanguageModelGeneratorContext<B: Backend> {
pub context: Rc<B::Context>,
pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
- pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- pub scratch_buffers: ScratchBuffers<B>,
+ pub shared_buffers: Rc<SharedBuffers<B>>,
pub model_config: LanguageModelConfig,
- pub decoder_config: DecoderConfig,
pub model_shape: ModelShape,
pub executables: Decoder<B>,
pub kv_cache_update: Box<KVCacheUpdate<B>>,
@@ -127,7 +112,6 @@ pub struct LanguageModelGeneratorContext<B: Backend> {
/// Kernels for copying sampled tokens in async pipeline
pub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
- pub token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,
/// Pre-allocated buffers for async generation
pub async_buffers: AsyncBuffers<B>,
}
@@ -159,10 +143,9 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let root_loader_view = loader.tree();
- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));
- shared_buffers.borrow_mut().update_data(&root_loader_view);
-
- let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_suffix_length);
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
+ shared_buffers.update_data(&root_loader_view);
+ let shared_buffers = Rc::new(shared_buffers);
let executables = Decoder::new(context.as_ref(), &decoder_config, &root_loader_view);
@@ -180,13 +163,10 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
);
let gpu_sampler =
- Sampling::<B>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)
- .map_err(|e| Error::UnableToCreateContext(e.into()))?;
+ Sampling::<B>::new(&context, intermediate_data_type).map_err(|e| Error::UnableToCreateContext(e.into()))?;
let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
.map_err(|e| Error::UnableToCreateContext(e.into()))?;
- let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)
- .map_err(|e| Error::UnableToCreateContext(e.into()))?;
let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());
let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);
@@ -197,16 +177,13 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
context,
cache_layers,
shared_buffers,
- scratch_buffers,
model_config: language_model_config.clone(),
- decoder_config,
model_shape,
executables,
kv_cache_update,
gpu_sampler,
seed,
token_copy_sampled,
- token_copy_results,
async_buffers,
};
diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rs
index 8596115c1..bbfa77026 100644
--- a/crates/backend-uzu/src/lib.rs
+++ b/crates/backend-uzu/src/lib.rs
@@ -25,6 +25,7 @@ pub mod session;
pub use array::{Array, ArrayContextExt};
#[cfg(metal_backend)]
pub use audio::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};
+pub use backends::common::{AllocationAccessError, allocation_copy_from_slice, allocation_to_vec};
pub use config::ConfigDataType;
pub use data_type::{ArrayElement, DataType};
pub use language_model::gumbel::{gumbel_float, revidx};
diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rs
index 620e19b5a..9bf5c6e86 100644
--- a/crates/backend-uzu/src/parameters/loader.rs
+++ b/crates/backend-uzu/src/parameters/loader.rs
@@ -9,7 +9,7 @@ use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError
use crate::{
DataType,
array::{Array, ArrayContextExt},
- backends::common::{Backend, Buffer, Context, DenseBuffer},
+ backends::common::{Allocation, AllocationType, AsBufferRangeRef, Backend, Context, DenseBuffer},
utils::fs::file_read_exact_at,
};
@@ -94,27 +94,23 @@ where
self.index.keys()
}
- pub fn get_leaf<'leaf>(
+ fn get_leaf<'leaf>(
&'leaf self,
key: &str,
) -> Result<ParameterLeaf<'file, 'context, 'leaf, C>, ParameterLoaderError<C::Backend>> {
Ok(ParameterLeaf {
- key: key.to_string(),
metadata: self.index.get(key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.to_string()))?,
loader: self,
})
}
- pub fn get(
+ fn get(
&self,
key: &str,
) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {
let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
let (offset, size) = (metadata_entry.offset, metadata_entry.size);
- let array_key = key.replace(".", "_");
- let array_label = format!("parameter_loader_{array_key}");
- let mut array =
- self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type, &array_label);
+ let mut array = self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type);
if array.size() != size {
return Err(ParameterLoaderError::SizeMismatch {
data_type: metadata_entry.data_type,
@@ -151,7 +147,6 @@ where
}
pub struct ParameterLeaf<'file, 'context, 'leaf, C: Context> {
- key: String,
metadata: &'leaf ParameterMetadata,
loader: &'leaf ParameterLoader<'context, 'file, C>,
}
@@ -169,16 +164,25 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C
self.metadata.size
}
- pub fn read_buffer(&self) -> Result<<C::Backend as Backend>::DenseBuffer, ParameterLoaderError<C::Backend>> {
- let mut buffer =
- self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;
- buffer.set_label(Some(&format!("parameter_loader_{}", self.key.replace(".", "_"))));
+ pub fn read_allocation(&self) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {
+ let allocation = self
+ .loader
+ .context
+ .create_allocation(self.metadata.size, AllocationType::Global)
+ .map_err(ParameterLoaderError::BackendError)?;
+ let buffer_range = allocation.as_buffer_range_ref();
+ let range = buffer_range.range();
file_read_exact_at(
self.loader.file,
- unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, self.metadata.size) },
+ unsafe {
+ std::slice::from_raw_parts_mut(
+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),
+ range.len(),
+ )
+ },
self.metadata.offset as u64,
)?;
- Ok(buffer)
+ Ok(allocation)
}
}
@@ -188,13 +192,6 @@ pub struct ParameterTree<'loader, C: Context> {
}
impl<'loader, C: Context> ParameterTree<'loader, C> {
- pub fn new(loader: &'loader ParameterLoader<'loader, 'loader, C>) -> Self {
- Self {
- loader,
- prefix: None,
- }
- }
-
pub fn path_prefix(&self) -> Option<&str> {
self.prefix.as_deref()
}
@@ -236,6 +233,13 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
self.loader.get_leaf(&self.join_prefix(name))
}
+ pub fn leaf_allocation(
+ &self,
+ name: &str,
+ ) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {
+ self.leaf(name)?.read_allocation()
+ }
+
pub fn read_extract_at(
&self,
name: &str,
@@ -247,14 +251,14 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
}
}
-pub fn resolve_subtree<'tree, C: Context>(
+pub fn try_resolve_subtree<'tree, C: Context>(
tree: &'tree ParameterTree<C>,
candidates: &[&str],
-) -> ParameterTree<'tree, C> {
+) -> Result<ParameterTree<'tree, C>, ParameterLoaderError<C::Backend>> {
for candidate in candidates {
if let Ok(subtree) = tree.subtree(candidate) {
- return subtree;
+ return Ok(subtree);
}
}
- panic!("Could not find any of {:?} in parameter tree", candidates);
+ Err(ParameterLoaderError::SubtreeNotFound(candidates.join(" or ")))
}
diff --git a/crates/backend-uzu/src/parameters/mod.rs b/crates/backend-uzu/src/parameters/mod.rs
index 67e9f3eaa..a3b5e9087 100644
--- a/crates/backend-uzu/src/parameters/mod.rs
+++ b/crates/backend-uzu/src/parameters/mod.rs
@@ -3,5 +3,5 @@ mod safetensors_metadata;
// Re-export the safetensors header reader so other modules (e.g. decoder
// runner) can estimate parameter memory before creating a Context.
-pub use loader::{ParameterLeaf, ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};
+pub use loader::{ParameterLeaf, ParameterLoader, ParameterLoaderError, ParameterTree, try_resolve_subtree};
pub use safetensors_metadata::{HeaderLoadingError, read_metadata as read_safetensors_metadata};
diff --git a/crates/backend-uzu/src/session/chat_session.rs b/crates/backend-uzu/src/session/chat_session.rs
index b9088bd4e..7915ae225 100644
--- a/crates/backend-uzu/src/session/chat_session.rs
+++ b/crates/backend-uzu/src/session/chat_session.rs
@@ -460,8 +460,9 @@ impl ChatSession {
next_to_submit = batch_end;
for _ in 0..batch_submitted {
- let (_, token, duration_nanos) = receiver.recv().map_err(|_| Error::SamplingFailed)?;
+ let (idx, token, duration_nanos) = receiver.recv().map_err(|_| Error::SamplingFailed)?;
in_flight -= 1;
+ llm.finish_async(idx);
let duration = nanos_to_secs(duration_nanos);
llm.tokens_push(token);
@@ -494,7 +495,8 @@ impl ChatSession {
if should_stop {
while in_flight > 0 {
- let _ = receiver.recv();
+ let (idx, _, _) = receiver.recv().map_err(|_| Error::SamplingFailed)?;
+ llm.finish_async(idx);
in_flight -= 1;
}
if let Some(slice) = cache_slice {
diff --git a/crates/backend-uzu/src/session/classification_session.rs b/crates/backend-uzu/src/session/classification_session.rs
index f863ab7f4..fd243f4dd 100644
--- a/crates/backend-uzu/src/session/classification_session.rs
+++ b/crates/backend-uzu/src/session/classification_session.rs
@@ -13,10 +13,6 @@ use crate::{
};
pub struct ClassificationSession {
- #[allow(dead_code)]
- model_path: PathBuf,
- #[allow(dead_code)]
- model_metadata: ModelMetadata,
tokenizer: Tokenizer,
classifier: Box<dyn ClassifierTrait>,
input_processor: Box<dyn InputProcessor>,
@@ -50,8 +46,6 @@ impl ClassificationSession {
let classifier = Box::new(Classifier::<B>::new(&model_path, &model_metadata)?);
Ok(Self {
- model_path,
- model_metadata,
tokenizer,
classifier,
input_processor,
diff --git a/crates/backend-uzu/src/session/config/decoding_config.rs b/crates/backend-uzu/src/session/config/decoding_config.rs
index d2042126d..27fbd1d57 100644
--- a/crates/backend-uzu/src/session/config/decoding_config.rs
+++ b/crates/backend-uzu/src/session/config/decoding_config.rs
@@ -11,7 +11,6 @@ pub struct DecodingConfig {
pub speculator_config: SpeculatorConfig,
pub sampling_seed: SamplingSeed,
pub async_batch_size: AsyncBatchSize,
- pub allow_pre_encode: bool,
}
impl DecodingConfig {
@@ -22,7 +21,6 @@ impl DecodingConfig {
speculator_config: SpeculatorConfig,
sampling_seed: SamplingSeed,
async_batch_size: AsyncBatchSize,
- allow_pre_encode: bool,
) -> Self {
Self {
context_mode,
@@ -31,7 +29,6 @@ impl DecodingConfig {
speculator_config,
sampling_seed,
async_batch_size,
- allow_pre_encode,
}
}
@@ -49,7 +46,6 @@ impl Default for DecodingConfig {
speculator_config: SpeculatorConfig::default(),
sampling_seed: SamplingSeed::default(),
async_batch_size: AsyncBatchSize::default(),
- allow_pre_encode: true,
}
}
}
@@ -114,14 +110,4 @@ impl DecodingConfig {
..self.clone()
}
}
-
- pub fn with_allow_pre_encode(
- &self,
- allow_pre_encode: bool,
- ) -> Self {
- Self {
- allow_pre_encode,
- ..self.clone()
- }
- }
}
diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rs
index b1559d1a3..b536d3411 100644
--- a/crates/backend-uzu/src/session/tts_session.rs
+++ b/crates/backend-uzu/src/session/tts_session.rs
@@ -25,7 +25,6 @@ use tokenizers::Tokenizer;
use crate::{
DataType,
- array::ArrayContextExt,
audio::{
AudioCodecRuntime, AudioGenerationContext, AudioPcmBatch, AudioTokenGrid,
nanocodec::{AudioDecodeStepStats, AudioDecodeStreamState},
@@ -33,20 +32,14 @@ use crate::{
backends::common::{
Backend, Context as BackendContext, Encoder, Kernels,
kernel::{
- EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,
- TokenCopyToResultsKernel,
+ EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TokenCopySampledKernel,
kv_cache_update::KVCacheUpdate,
matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
},
},
config::{InnerModelConfig, ModelMetadata, TtsMessageProcessorConfig},
encodable_block::{Decoder, EncodingParameters, Sampling as GpuSampling},
- forward_pass::{
- cache_layers::CacheLayers,
- model_shape::ModelShape,
- scratch_buffers::ScratchBuffers,
- state::{ArrayId, ForwardPassState, SharedBuffers},
- },
+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},
parameters::ParameterLoader,
session::{
config::{TextDecoderRuntimeConfig, TextSamplingConfig, TtsChunkPolicy, TtsRunConfig, TtsSessionOptions},
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index d4f1bee74..c35872969 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -1,13 +1,17 @@
use super::{decoder_support::*, *};
-use crate::{array::Array, backends::common::DenseBuffer, session::types::TtsModelConfigError};
+use crate::{
+ array::{Array, ArrayContextExt},
+ backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},
+ encodable_block::{DecoderArguments, DecoderDecodeInput, SamplingInputs},
+ forward_pass::token_inputs::TokenInputs,
+ session::types::TtsModelConfigError,
+};
struct TokenDecoderLoadedModel<B: Backend> {
- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- scratch_buffers: ScratchBuffers<B>,
+ shared_buffers: Rc<SharedBuffers<B>>,
executables: Decoder<B>,
sampler: GpuSampling<B>,
token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
- token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,
}
impl<B: Backend> TokenDecoderLoadedModel<B> {
@@ -19,7 +23,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
transformer_subtree: &str,
embedding_subtree: &str,
readout_subtree: &str,
- max_suffix_length: usize,
) -> Result<Self, Error> {
let weights_path = model_path.join("model.safetensors");
let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
@@ -33,7 +36,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
&root_loader_view,
transformer_subtree,
)?;
- let scratch_buffers = ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_suffix_length);
let executables = Decoder::new_with_embedding_and_readout_subtrees(
context.as_ref(),
&decoder_config,
@@ -42,22 +44,16 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
embedding_subtree,
readout_subtree,
);
- let logits_data_type = scratch_buffers.logits.data_type();
- let sampler =
- GpuSampling::new(context.as_ref(), logits_data_type, max_suffix_length, decoder_config.vocab_size)
- .map_err(unable_to_create_context)?;
+ let logits_data_type = model_shape.activation_data_type();
+ let sampler = GpuSampling::new(context.as_ref(), logits_data_type).map_err(unable_to_create_context)?;
let token_copy_sampled =
<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;
- let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(context.as_ref())
- .map_err(unable_to_create_context)?;
Ok(Self {
shared_buffers,
- scratch_buffers,
executables,
sampler,
token_copy_sampled,
- token_copy_results,
})
}
}
@@ -65,8 +61,7 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
struct TokenDecoderContext<B: Backend> {
context: Rc<B::Context>,
cache_layers: Rc<RefCell<CacheLayers<B>>>,
- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
- scratch_buffers: ScratchBuffers<B>,
+ shared_buffers: Rc<SharedBuffers<B>>,
model_shape: ModelShape,
decoder_config: Rc<crate::config::DecoderConfig>,
runtime_config: TextDecoderRuntimeConfig,
@@ -74,10 +69,9 @@ struct TokenDecoderContext<B: Backend> {
sampler: GpuSampling<B>,
kv_cache_update: KVCacheUpdate<B>,
token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
- token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,
- async_chain_positions: Rc<RefCell<B::DenseBuffer>>,
- async_chain_seeds: Rc<RefCell<B::DenseBuffer>>,
- async_chain_results: Rc<RefCell<B::DenseBuffer>>,
+ async_chain_positions: Array<B>,
+ async_chain_seeds: Array<B>,
+ async_chain_results: Allocation<B>,
async_chain_capacity: usize,
current_max_prefix_length: usize,
}
@@ -108,7 +102,6 @@ impl<B: Backend> TokenDecoderContext<B> {
transformer_subtree,
embedding_subtree,
readout_subtree,
- max_suffix_length,
)?;
let async_chain_capacity = max_suffix_length.max(1);
let (async_chain_positions, async_chain_seeds, async_chain_results) =
@@ -128,7 +121,6 @@ impl<B: Backend> TokenDecoderContext<B> {
context,
cache_layers,
shared_buffers: loaded_model.shared_buffers,
- scratch_buffers: loaded_model.scratch_buffers,
model_shape,
decoder_config,
runtime_config: runtime_config.clone(),
@@ -136,7 +128,6 @@ impl<B: Backend> TokenDecoderContext<B> {
sampler: loaded_model.sampler,
kv_cache_update,
token_copy_sampled: loaded_model.token_copy_sampled,
- token_copy_results: loaded_model.token_copy_results,
async_chain_positions,
async_chain_seeds,
async_chain_results,
@@ -153,48 +144,34 @@ impl<B: Backend> TokenDecoderContext<B> {
model_shape: &ModelShape,
root_loader_view: &crate::parameters::ParameterTree<B::Context>,
transformer_subtree: &str,
- ) -> Result<Rc<RefCell<SharedBuffers<B>>>, Error> {
- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), decoder_config, model_shape)));
+ ) -> Result<Rc<SharedBuffers<B>>, Error> {
+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config, model_shape);
let transformer_tree = root_loader_view.subtree(transformer_subtree).map_err(|_| Error::UnableToLoadWeights)?;
- let mut shared_buffers_borrow = shared_buffers.borrow_mut();
- if let Some(global_rope) = &mut shared_buffers_borrow.global_rope {
+ if let Some(global_rope) = &mut shared_buffers.global_rope {
global_rope.update_data(&transformer_tree, "global_rope");
}
- if let Some(local_rope) = &mut shared_buffers_borrow.local_rope {
+ if let Some(local_rope) = &mut shared_buffers.local_rope {
local_rope.update_data(&transformer_tree, "local_rope");
}
- drop(shared_buffers_borrow);
- Ok(shared_buffers)
+ Ok(Rc::new(shared_buffers))
}
fn build_async_chain_buffers(
context: &Rc<B::Context>,
async_chain_capacity: usize,
- ) -> Result<(Rc<RefCell<B::DenseBuffer>>, Rc<RefCell<B::DenseBuffer>>, Rc<RefCell<B::DenseBuffer>>), Error> {
- let positions = Rc::new(RefCell::new(
- context
- .create_buffer(async_chain_capacity * std::mem::size_of::<i32>())
- .map_err(unable_to_create_context)?,
- ));
- let seeds = Rc::new(RefCell::new(
- context
- .create_buffer(async_chain_capacity * std::mem::size_of::<u64>())
- .map_err(unable_to_create_context)?,
- ));
- let results = Rc::new(RefCell::new(
- context
- .create_buffer(async_chain_capacity * std::mem::size_of::<u32>())
- .map_err(unable_to_create_context)?,
- ));
+ ) -> Result<(Array<B>, Array<B>, Allocation<B>), Error> {
+ let positions = context.create_array_uninitialized(&[async_chain_capacity], crate::DataType::I32);
+ let seeds = context.create_array_uninitialized(&[async_chain_capacity], crate::DataType::U64);
+ let results =
+ context.create_array_uninitialized(&[async_chain_capacity], crate::DataType::U32).into_allocation();
Ok((positions, seeds, results))
}
}
pub(super) struct TokenDecoderRunner<B: Backend> {
ctx: TokenDecoderContext<B>,
- pub(super) single_hidden_capture: Array<B>,
- pub(super) single_override_embedding: Array<B>,
- tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,
+ pub(super) single_hidden_capture: Allocation<B>,
+ pub(super) single_override_embedding: Allocation<B>,
tensor_add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,
single_token_vocab_masks: HashMap<usize, Box<[u32]>>,
two_token_vocab_masks: HashMap<usize, Box<[u32]>>,
@@ -203,6 +180,10 @@ pub(super) struct TokenDecoderRunner<B: Backend> {
}
impl<B: Backend> TokenDecoderRunner<B> {
+ pub(super) fn activation_data_type(&self) -> DataType {
+ self.ctx.model_shape.activation_data_type()
+ }
+
pub(super) fn context(&self) -> &Rc<B::Context> {
self.ctx.context()
}
@@ -227,21 +208,17 @@ impl<B: Backend> TokenDecoderRunner<B> {
)?;
let context: Rc<B::Context> = Rc::clone(ctx.context());
let model_dim = ctx.decoder_config.model_dim;
- let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(context.as_ref(), activation_data_type)
- .map_err(unable_to_create_context)?;
let tensor_add_scale =
<B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type)
.map_err(unable_to_create_context)?;
- let single_hidden_capture =
- context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");
+ let single_hidden_capture = context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();
let single_override_embedding =
- context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_override_embedding");
+ context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();
Ok(Self {
ctx,
single_hidden_capture,
single_override_embedding,
- tensor_copy,
tensor_add_scale,
single_token_vocab_masks: HashMap::new(),
two_token_vocab_masks: HashMap::new(),
@@ -251,10 +228,110 @@ impl<B: Backend> TokenDecoderRunner<B> {
}
pub(super) fn reset(&mut self) {
- self.ctx.cache_layers.borrow_mut().clear();
+ self.ctx.cache_layers.borrow_mut().clear(self.ctx.context.as_ref());
self.next_position = 0;
}
+ fn create_sampling_inputs_from_slices(
+ &self,
+ token_seeds: &[u64],
+ token_bitmask: Option<&[u32]>,
+ ) -> SamplingInputs<B> {
+ let bitmask_row_len = token_bitmask.map(|_| self.ctx.decoder_config.vocab_size.div_ceil(32));
+ SamplingInputs::from_slices(self.ctx.context.as_ref(), token_seeds, token_bitmask, bitmask_row_len)
+ }
+
+ fn llm_token_inputs(
+ &self,
+ token_ids: &[u64],
+ token_positions: &[usize],
+ token_ids_array: Option<Array<B>>,
+ token_positions_array: Option<Array<B>>,
+ sampling_start: usize,
+ sampling_length: usize,
+ ) -> TokenInputs<B> {
+ TokenInputs::new_llm(
+ self.ctx.context.as_ref(),
+ &self.ctx.model_shape,
+ token_ids,
+ None,
+ token_positions,
+ token_ids_array,
+ token_positions_array,
+ sampling_start,
+ sampling_length,
+ )
+ }
+
+ fn decoder_arguments<'a>(
+ &'a self,
+ token_inputs: &'a TokenInputs<B>,
+ batch_dim: usize,
+ sampling_start: usize,
+ sampling_length: usize,
+ cache_layers: &'a mut CacheLayers<B>,
+ ) -> DecoderArguments<'a, B> {
+ DecoderArguments {
+ token_positions: token_inputs.token_positions(),
+ token_parents: token_inputs.token_parents(),
+ token_subtrie_ranges: token_inputs.token_subtrie_ranges(),
+ shared_buffers: self.ctx.shared_buffers.as_ref(),
+ cache_layers: Some(cache_layers),
+ batch_dim,
+ sampling_start,
+ sampling_length,
+ rope_max_sequence_length: self.ctx.model_shape.context_length(),
+ rope_dim: self.ctx.model_shape.rope_dim(),
+ #[cfg(feature = "tracing")]
+ trace: None,
+ }
+ }
+
+ fn encode_sampling_on(
+ &self,
+ encoder: &mut Encoder<B>,
+ logits: &mut Allocation<B>,
+ output: &mut Allocation<B>,
+ sampling_method: SamplingMethod,
+ batch_size: usize,
+ sampling_inputs: SamplingInputs<B>,
+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
+ ) -> Result<(), Error> {
+ pending_sampling_inputs.push(sampling_inputs);
+ let sampling_inputs = pending_sampling_inputs.last().expect("sampling inputs must be pending");
+ self.ctx
+ .sampler
+ .encode(
+ crate::encodable_block::SamplingArguments {
+ logits,
+ seeds: sampling_inputs.seeds.allocation(),
+ bitmask: sampling_inputs.bitmask.as_ref().map(Array::allocation),
+ output,
+ sampling_method,
+ batch_size,
+ vocab_size: self.ctx.decoder_config.vocab_size,
+ },
+ encoder,
+ )
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))
+ }
+
+ fn copy_sampling_output_to_async_result_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ sampling_output: &Allocation<B>,
+ result_slot: usize,
+ ) {
+ let copy_size = std::mem::size_of::<u32>();
+ let result_offset = result_slot * copy_size;
+ encoder.encode_copy(
+ sampling_output,
+ 0..copy_size,
+ &mut self.ctx.async_chain_results,
+ result_offset..result_offset + copy_size,
+ );
+ }
+
pub(super) fn prepare_for_generation(
&mut self,
max_prefix_length: usize,
@@ -286,44 +363,24 @@ impl<B: Backend> TokenDecoderRunner<B> {
let token_count = token_ids.len();
let positions: Vec<usize> = (self.next_position..self.next_position + token_count).collect();
- let token_seeds = vec![0_u64; token_count];
-
- let mut state = ForwardPassState::new_llm(
- self.ctx.context.clone(),
- &self.ctx.decoder_config,
- &self.ctx.model_shape,
- &self.ctx.scratch_buffers,
- self.ctx.cache_layers.clone(),
- self.ctx.shared_buffers.clone(),
- token_ids,
- None,
- &positions,
- None, // no bitmask
- &token_seeds,
- token_count,
- 0, // sampling_start: irrelevant
- 0, // sampling_length: no sampling
- false,
- false,
- None,
- None,
- );
+ let token_inputs = self.llm_token_inputs(token_ids, &positions, None, None, 0, 0);
let encoding_parameters = EncodingParameters::new();
let context = Rc::clone(&self.ctx.context);
let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
+ let mut cache_layers = self.ctx.cache_layers.borrow_mut();
self.ctx
.executables
- .embed
- .encode_lookup(&mut state, &mut encoder)
+ .encode_prefill(
+ self.decoder_arguments(&token_inputs, token_count, 0, 0, &mut *cache_layers),
+ token_inputs.token_ids(),
+ &encoding_parameters,
+ &mut encoder,
+ )
.map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- for layer in self.ctx.executables.layers.iter() {
- layer
- .encode(&mut state, &encoding_parameters, &mut encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- }
+ drop(cache_layers);
self.encode_cache_acceptance_update_on(&mut encoder, token_count);
self.submit_and_wait_command_buffer(encoder)?;
@@ -354,10 +411,13 @@ impl<B: Backend> TokenDecoderRunner<B> {
encoder: &mut Encoder<B>,
initial_token_ids: &[u64],
initial_embedding_injection: EmbeddingInjection,
- pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,
followup_count: usize,
vocab_limit: Option<usize>,
sampling: &mut TextSamplingState,
+ pending_token_inputs: &mut Vec<TokenInputs<B>>,
+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
+ pending_sampling_outputs: &mut Vec<Allocation<B>>,
) -> Result<usize, Error> {
let total_count =
1_usize.checked_add(followup_count).ok_or(TtsModelConfigError::AsyncChainFollowupCountOverflow)?;
@@ -416,34 +476,24 @@ impl<B: Backend> TokenDecoderRunner<B> {
// Pre-fill followup positions and seeds into the async chain buffers.
let followup_base_position = self.next_position + initial_token_ids.len();
{
- let positions_borrow = self.ctx.async_chain_positions.borrow();
- let positions_ptr = positions_borrow.cpu_ptr().as_ptr() as *mut i32;
+ let positions = self.ctx.async_chain_positions.as_slice_mut::<i32>();
for pass in 0..followup_count {
- unsafe {
- *positions_ptr.add(pass) = (followup_base_position + pass) as i32;
- }
+ positions[pass] = (followup_base_position + pass) as i32;
}
}
{
- let seeds_borrow = self.ctx.async_chain_seeds.borrow();
- let seeds_ptr = seeds_borrow.cpu_ptr().as_ptr() as *mut u64;
+ let seeds = self.ctx.async_chain_seeds.as_slice_mut::<u64>();
if !matches!(sampling.method(), SamplingMethod::Greedy) {
- for pass in 0..followup_count {
- unsafe {
- *seeds_ptr.add(pass) = sampling.next_seed();
- }
+ for seed in seeds.iter_mut().take(followup_count) {
+ *seed = sampling.next_seed();
}
} else {
- for pass in 0..followup_count {
- unsafe {
- *seeds_ptr.add(pass) = 0;
- }
- }
+ seeds[..followup_count].fill(0);
}
}
// Encode the initial forward pass (no new CB, no submit).
- let state = self.encode_single_forward_pass_on(
+ let sampling_output = self.encode_single_forward_pass_on(
encoder,
initial_token_ids,
initial_embedding_injection,
@@ -452,30 +502,41 @@ impl<B: Backend> TokenDecoderRunner<B> {
false, // capture_hidden
pre_injection_encode,
initial_seed,
+ pending_token_inputs,
+ pending_sampling_inputs,
)?;
- // Copy the sampled token for chaining and into results[0].
- {
- let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;
- let sampling_output_buffer = sampling_output.buffer();
- let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();
-
- let sampling_output_buffer = sampling_output_buffer.borrow();
- let mut token_ids_buffer = token_ids_buffer.borrow_mut();
- if followup_count > 0 {
- self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);
- }
- let mut results_buffer = self.ctx.async_chain_results.borrow_mut();
- self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);
- }
+ let first_followup_token_ids = if followup_count > 0 {
+ let token_ids_shape = [1];
+ let token_ids_data_type = crate::DataType::U64;
+ let mut next_token_ids =
+ self.ctx.context.create_array_uninitialized(&token_ids_shape, token_ids_data_type).into_allocation();
+ self.ctx.token_copy_sampled.encode(&sampling_output, &mut next_token_ids, encoder);
+ Some(unsafe { Array::from_allocation(next_token_ids, 0, &token_ids_shape, token_ids_data_type) })
+ } else {
+ None
+ };
+ self.copy_sampling_output_to_async_result_on(encoder, &sampling_output, 0);
self.encode_cache_acceptance_update_on(encoder, initial_token_count);
self.register_positions_and_advance(initial_token_count);
if followup_count > 0 {
- self.encode_followup_passes_on(encoder, followup_count, 0, vocab_mask_limit, sampling.method(), 1)?;
+ self.encode_followup_passes_on(
+ encoder,
+ followup_count,
+ first_followup_token_ids,
+ vocab_mask_limit,
+ sampling.method(),
+ 1,
+ pending_token_inputs,
+ pending_sampling_inputs,
+ pending_sampling_outputs,
+ )?;
}
+ pending_sampling_outputs.push(sampling_output);
+
self.next_position = followup_base_position.saturating_add(followup_count);
// Do NOT submit -- the caller will submit after encoding more work.
Ok(total_count)
@@ -493,91 +554,69 @@ impl<B: Backend> TokenDecoderRunner<B> {
&mut self,
encoder: &mut Encoder<B>,
followup_count: usize,
- first_token: u64,
+ first_token_ids: Option<Array<B>>,
vocab_mask_limit: Option<usize>,
sampling_method: SamplingMethod,
results_offset_slots: usize,
+ pending_token_inputs: &mut Vec<TokenInputs<B>>,
+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
+ pending_sampling_outputs: &mut Vec<Allocation<B>>,
) -> Result<(), Error> {
let encoding_parameters = EncodingParameters::new();
- let context_rc = self.ctx.context.clone();
let cache_layers_rc = self.ctx.cache_layers.clone();
- let shared_buffers_rc = self.ctx.shared_buffers.clone();
- let positions_rc = self.ctx.async_chain_positions.clone();
- let seeds_rc = self.ctx.async_chain_seeds.clone();
- let token_bitmask = vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit));
+ let token_bitmask =
+ vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit)).map(Box::<[u32]>::from);
+ let mut next_token_ids = first_token_ids;
for pass in 0..followup_count {
let results_slot = results_offset_slots + pass;
- let token_ids = [if pass == 0 {
- first_token
- } else {
- 0
- }];
- let mut state = ForwardPassState::new_llm(
- context_rc.clone(),
- &self.ctx.decoder_config,
- &self.ctx.model_shape,
- &self.ctx.scratch_buffers,
- cache_layers_rc.clone(),
- shared_buffers_rc.clone(),
- &token_ids,
- None,
- &[self.next_position + pass],
- token_bitmask,
- &[0],
- 1,
- 0,
- 1,
- false,
- pass > 0 || results_offset_slots > 0,
- Some((positions_rc.clone(), pass)),
- Some((seeds_rc.clone(), pass)),
- );
- if let Some(method) = state.sampling_method_mut() {
- *method = Some(sampling_method);
- }
-
- {
- self.ctx
- .executables
- .embed
- .encode_lookup(&mut state, encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- for layer in self.ctx.executables.layers.iter() {
- layer
- .encode(&mut state, &encoding_parameters, encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- }
- self.ctx
- .executables
- .norm
- .encode(&mut state, encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- self.ctx
- .executables
- .embed
- .encode_readout(&mut state, encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- }
-
- let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;
- let sampling_output_buffer = sampling_output.buffer();
- let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();
+ let token_ids = [0];
+ let seed = self.ctx.async_chain_seeds.as_slice::<u64>()[pass];
+ let sampling_inputs = self.create_sampling_inputs_from_slices(&[seed], token_bitmask.as_deref());
+ let token_inputs =
+ self.llm_token_inputs(&token_ids, &[self.next_position + pass], next_token_ids.take(), None, 0, 1);
+ pending_token_inputs.push(token_inputs);
+ let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");
{
- let sampling_output_buffer = sampling_output_buffer.borrow();
- let mut token_ids_buffer = token_ids_buffer.borrow_mut();
+ let mut logits = {
+ let mut cache_layers = cache_layers_rc.borrow_mut();
+ self.ctx
+ .executables
+ .encode_decode(
+ self.decoder_arguments(&token_inputs, 1, 0, 1, &mut *cache_layers),
+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),
+ None,
+ &encoding_parameters,
+ encoder,
+ )
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?
+ };
+ let mut sampling_output = self.create_sampling_output(1);
+ self.encode_sampling_on(
+ encoder,
+ &mut logits,
+ &mut sampling_output,
+ sampling_method,
+ 1,
+ sampling_inputs,
+ pending_sampling_inputs,
+ )?;
if pass + 1 < followup_count {
- self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);
+ let token_ids_shape = [1];
+ let token_ids_data_type = crate::DataType::U64;
+ let mut updated_token_ids = self
+ .ctx
+ .context
+ .create_array_uninitialized(&token_ids_shape, token_ids_data_type)
+ .into_allocation();
+ self.ctx.token_copy_sampled.encode(&sampling_output, &mut updated_token_ids, encoder);
+ next_token_ids = Some(unsafe {
+ Array::from_allocation(updated_token_ids, 0, &token_ids_shape, token_ids_data_type)
+ });
}
- let results_offset = results_slot * std::mem::size_of::<u32>();
- let mut results_buffer = self.ctx.async_chain_results.borrow_mut();
- self.ctx.token_copy_results.encode(
- &*sampling_output_buffer,
- (&mut *results_buffer, results_offset),
- encoder,
- );
+ self.copy_sampling_output_to_async_result_on(encoder, &sampling_output, results_slot);
+ pending_sampling_outputs.push(sampling_output);
}
self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);
@@ -647,6 +686,13 @@ impl<B: Backend> TokenDecoderRunner<B> {
self.two_token_vocab_masks.get(&vocab_limit).map(|mask| mask.as_ref())
}
+ fn create_sampling_output(
+ &self,
+ sampling_length: usize,
+ ) -> Allocation<B> {
+ self.ctx.context.create_array_uninitialized(&[sampling_length], crate::DataType::U32).into_allocation()
+ }
+
fn encode_single_forward_pass_on(
&mut self,
encoder: &mut Encoder<B>,
@@ -655,9 +701,11 @@ impl<B: Backend> TokenDecoderRunner<B> {
sampling: &mut TextSamplingState,
token_bitmask: Option<&[u32]>,
capture_hidden: bool,
- mut pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ mut pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,
preconsumed_seed: Option<u64>,
- ) -> Result<ForwardPassState<B>, Error> {
+ pending_token_inputs: &mut Vec<TokenInputs<B>>,
+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
+ ) -> Result<Allocation<B>, Error> {
if token_ids.is_empty() {
return Err(Error::GenerateFailed);
}
@@ -715,68 +763,80 @@ impl<B: Backend> TokenDecoderRunner<B> {
return Err(Error::GenerateFailed);
}
- let mut state = ForwardPassState::new_llm(
- self.ctx.context.clone(),
- &self.ctx.decoder_config,
- &self.ctx.model_shape,
- &self.ctx.scratch_buffers,
- self.ctx.cache_layers.clone(),
- self.ctx.shared_buffers.clone(),
- token_ids,
- None,
- positions,
- token_bitmask,
- token_seeds,
- token_count,
- sampling_start,
- sampling_length,
- false,
- false,
- None,
- None,
- );
- if let Some(method) = state.sampling_method_mut() {
- *method = Some(sampling.method());
- }
+ let sampling_seed_end = sampling_start + sampling_length;
+ let sampling_token_seeds = &token_seeds[sampling_start..sampling_seed_end];
+ let sampling_token_bitmask = token_bitmask.map(|mask| {
+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);
+ let row_start = sampling_start * row_words;
+ let row_end = row_start + sampling_length * row_words;
+ &mask[row_start..row_end]
+ });
+ let sampling_inputs = self.create_sampling_inputs_from_slices(sampling_token_seeds, sampling_token_bitmask);
+ let token_inputs = self.llm_token_inputs(token_ids, positions, None, None, sampling_start, sampling_length);
+ pending_token_inputs.push(token_inputs);
+ let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");
let encoding_parameters = EncodingParameters::new();
- self.ctx
+ let mut main = self
+ .ctx
.executables
.embed
- .encode_lookup(&mut state, encoder)
+ .encode_lookup(token_inputs.token_ids(), token_count, encoder)
.map_err(|err| Error::EncodeFailed(Box::new(err)))?;
if let Some(pre_encode) = pre_injection_encode.as_mut() {
- pre_encode(self, &state, encoder)?;
+ pre_encode(self, encoder)?;
}
match embedding_injection {
EmbeddingInjection::None => {},
EmbeddingInjection::AddPreloaded {
post_scale,
} => {
- self.encode_add_scale_from_single_bias_on(encoder, &state, token_count, post_scale.unwrap_or(1.0))?;
+ self.encode_add_scale_from_single_bias_on(encoder, &mut main, token_count, post_scale.unwrap_or(1.0))?;
},
EmbeddingInjection::OverrideFirstRowInternal => {
- self.encode_override_first_row_from_device_on(encoder, &state, &self.single_override_embedding)?;
+ self.encode_override_first_row_from_device_on(encoder, &mut main, &self.single_override_embedding)?;
},
}
- for layer in self.ctx.executables.layers.iter() {
- layer
- .encode(&mut state, &encoding_parameters, encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- }
- self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- if capture_hidden {
- // After output norm with residual_add, Shortcut holds the un-normalized full residual
- self.encode_capture_last_hidden_into_single_buffer_on(encoder, &state, token_count)?;
- }
- self.ctx
- .executables
- .embed
- .encode_readout(&mut state, encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
- self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ let mut logits = {
+ let mut cache_layers = self.ctx.cache_layers.borrow_mut();
+ let decoder_arguments = DecoderArguments {
+ token_positions: token_inputs.token_positions(),
+ token_parents: token_inputs.token_parents(),
+ token_subtrie_ranges: token_inputs.token_subtrie_ranges(),
+ shared_buffers: self.ctx.shared_buffers.as_ref(),
+ cache_layers: Some(&mut *cache_layers),
+ batch_dim: token_count,
+ sampling_start,
+ sampling_length,
+ rope_max_sequence_length: self.ctx.model_shape.context_length(),
+ rope_dim: self.ctx.model_shape.rope_dim(),
+ #[cfg(feature = "tracing")]
+ trace: None,
+ };
+ let hidden_capture = capture_hidden.then_some(&mut self.single_hidden_capture);
+ self.ctx
+ .executables
+ .encode_decode(
+ decoder_arguments,
+ DecoderDecodeInput::Embeddings(main),
+ hidden_capture,
+ &encoding_parameters,
+ encoder,
+ )
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?
+ };
+ let mut sampling_output = self.create_sampling_output(sampling_length);
+ self.encode_sampling_on(
+ encoder,
+ &mut logits,
+ &mut sampling_output,
+ sampling.method(),
+ sampling_length,
+ sampling_inputs,
+ pending_sampling_inputs,
+ )?;
- Ok(state)
+ Ok(sampling_output)
}
/// Encode the KV-cache acceptance update for `token_count` tokens onto
@@ -835,7 +895,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
sampling: &mut TextSamplingState,
precomputed_token_bitmask: Option<&[u32]>,
capture_hidden: bool,
- pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,
) -> Result<u64, Error> {
objc2::rc::autoreleasepool(|_| {
if token_ids.is_empty() {
@@ -910,8 +970,10 @@ impl<B: Backend> TokenDecoderRunner<B> {
let context = Rc::clone(&self.ctx.context);
let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
+ let mut pending_token_inputs = Vec::new();
+ let mut pending_sampling_inputs = Vec::new();
- let state = self.encode_single_forward_pass_on(
+ let sampling_output = self.encode_single_forward_pass_on(
&mut encoder,
token_ids,
embedding_injection,
@@ -920,97 +982,38 @@ impl<B: Backend> TokenDecoderRunner<B> {
capture_hidden,
pre_injection_encode,
None,
+ &mut pending_token_inputs,
+ &mut pending_sampling_inputs,
)?;
self.encode_cache_acceptance_update_on(&mut encoder, token_count);
self.submit_and_wait_command_buffer(encoder)?;
- let token = read_sampled_token_from_sampling_output(&state)?;
+ let token = read_sampled_token_from_sampling_output(&sampling_output)?;
self.register_positions_and_advance(token_count);
Ok(token)
})
}
- fn encode_capture_last_hidden_into_single_buffer_on(
- &self,
- encoder: &mut Encoder<B>,
- state: &ForwardPassState<B>,
- token_count: usize,
- ) -> Result<(), Error> {
- if token_count == 0 {
- return Err(Error::GenerateFailed);
- }
- let model_dim = self.ctx.decoder_config.model_dim;
- let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
- model_dim,
- })?;
- let shortcut = state.array(ArrayId::Shortcut);
- let bytes_per_element = shortcut.data_type().size_in_bytes();
- let row_offset = (token_count - 1)
- .checked_mul(model_dim)
- .and_then(|value| value.checked_mul(bytes_per_element))
- .ok_or(TtsModelConfigError::HiddenCaptureRowOffsetOverflow {
- token_count,
- model_dim,
- })?;
- let src_offset =
- shortcut.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;
- let capture = &self.single_hidden_capture;
- if capture.shape() != [1, model_dim] || capture.data_type() != shortcut.data_type() {
- return Err(TtsModelConfigError::HiddenCaptureTensorMismatch {
- expected_shape: [1, model_dim].into(),
- expected_data_type: shortcut.data_type(),
- actual_shape: capture.shape().into(),
- actual_data_type: capture.data_type(),
- }
- .into());
- }
-
- let shortcut_buffer = shortcut.buffer();
- let shortcut_buffer = shortcut_buffer.borrow();
- let capture_buffer = capture.buffer();
- let mut capture_buffer = capture_buffer.borrow_mut();
- self.tensor_copy.encode((&*shortcut_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);
- Ok(())
- }
-
fn encode_override_first_row_from_device_on(
&self,
encoder: &mut Encoder<B>,
- state: &ForwardPassState<B>,
- override_embedding: &Array<B>,
+ main: &mut Allocation<B>,
+ override_embedding: &Allocation<B>,
) -> Result<(), Error> {
let model_dim = self.ctx.decoder_config.model_dim;
- let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
- model_dim,
- })?;
- let main = state.array(ArrayId::Main);
- if override_embedding.shape() != [1, model_dim] || override_embedding.data_type() != main.data_type() {
- return Err(TtsModelConfigError::OverrideEmbeddingTensorMismatch {
- expected_shape: [1, model_dim].into(),
- expected_data_type: main.data_type(),
- actual_shape: override_embedding.shape().into(),
- actual_data_type: override_embedding.data_type(),
- }
- .into());
- }
-
- let override_buffer = override_embedding.buffer();
- let override_buffer = override_buffer.borrow();
- let main_buffer = main.buffer();
- let mut main_buffer = main_buffer.borrow_mut();
- self.tensor_copy.encode(
- (&*override_buffer, override_embedding.offset()),
- (&mut *main_buffer, main.offset()),
- model_dim_u32,
- encoder,
- );
+ let model_dim_bytes = model_dim.checked_mul(self.activation_data_type().size_in_bytes()).ok_or(
+ TtsModelConfigError::ModelDimExceedsU32 {
+ model_dim,
+ },
+ )?;
+ encoder.encode_copy(override_embedding, 0..model_dim_bytes, main, 0..model_dim_bytes);
Ok(())
}
fn encode_add_scale_from_single_bias_on(
&self,
encoder: &mut Encoder<B>,
- state: &ForwardPassState<B>,
+ main: &mut Allocation<B>,
token_count: usize,
scale: f32,
) -> Result<(), Error> {
@@ -1029,34 +1032,18 @@ impl<B: Backend> TokenDecoderRunner<B> {
u32::try_from(total_len).map_err(|_| TtsModelConfigError::AddScaleTotalLengthExceedsU32 {
total_len,
})?;
-
- let main = state.array(ArrayId::Main);
- let bias = &self.single_override_embedding;
- if bias.shape() != [1, model_dim] || bias.data_type() != main.data_type() {
- return Err(TtsModelConfigError::AddScaleBiasTensorMismatch {
- expected_shape: [1, model_dim].into(),
- expected_data_type: main.data_type(),
- actual_shape: bias.shape().into(),
- actual_data_type: bias.data_type(),
- }
- .into());
- }
-
- let bias_buffer = bias.buffer();
- let bias_buffer = bias_buffer.borrow();
- let main_output_buffer = main.buffer();
- let mut main_output_buffer = main_output_buffer.borrow_mut();
- // TensorAddScale is elementwise, so in-place read/write aliasing is valid here.
- let main_input_buffer: &B::DenseBuffer = unsafe { &*(&*main_output_buffer as *const B::DenseBuffer) };
+ let mut output =
+ encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(unable_to_create_context)?;
self.tensor_add_scale.encode(
- (main_input_buffer, main.offset()),
- &*bias_buffer,
- (&mut *main_output_buffer, main.offset()),
+ &*main,
+ &self.single_override_embedding,
+ &mut output,
model_dim_u32,
total_len_u32,
scale,
encoder,
);
+ *main = output;
Ok(())
}
@@ -1075,11 +1062,14 @@ impl<B: Backend> TokenDecoderRunner<B> {
sampling: &mut TextSamplingState,
precomputed_token_bitmask: Option<&[u32]>,
capture_hidden: bool,
- pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,
+ pending_token_inputs: &mut Vec<TokenInputs<B>>,
+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
+ pending_sampling_outputs: &mut Vec<Allocation<B>>,
) -> Result<(), Error> {
let token_count = token_ids.len();
- let state = self.encode_single_forward_pass_on(
+ let sampling_output = self.encode_single_forward_pass_on(
encoder,
token_ids,
embedding_injection,
@@ -1088,18 +1078,14 @@ impl<B: Backend> TokenDecoderRunner<B> {
capture_hidden,
pre_injection_encode,
None,
+ pending_token_inputs,
+ pending_sampling_inputs,
)?;
// Copy sampled token to async_chain_results[0] so the caller can
// read it after the eventual submit.
- {
- let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;
- let sampling_output_buffer = sampling_output.buffer();
-
- let sampling_output_buffer = sampling_output_buffer.borrow();
- let mut results_buffer = self.ctx.async_chain_results.borrow_mut();
- self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);
- }
+ self.copy_sampling_output_to_async_result_on(encoder, &sampling_output, 0);
+ pending_sampling_outputs.push(sampling_output);
self.encode_cache_acceptance_update_on(encoder, token_count);
// Do NOT submit -- the caller will submit.
@@ -1112,20 +1098,22 @@ impl<B: Backend> TokenDecoderRunner<B> {
encoder: Encoder<B>,
) -> Result<(), Error> {
self.instrumentation.command_buffers_submitted += 1;
- encoder
+ let result = encoder
.end_encoding()
.submit()
.wait_until_completed()
- .map_err(|err| Error::CommandBufferFailed(Box::new(err)))?;
+ .map_err(|err| Error::CommandBufferFailed(Box::new(err)));
+ result?;
self.instrumentation.host_waits += 1;
Ok(())
}
pub(super) fn copy_async_chain_results_to_on(
- &self,
+ &mut self,
encoder: &mut Encoder<B>,
src_slot: usize,
- dst: (&mut B::DenseBuffer, usize),
+ dst: &mut Allocation<B>,
+ dst_offset: usize,
count: usize,
) -> Result<(), Error> {
let src_offset = src_slot.checked_mul(std::mem::size_of::<u32>()).ok_or(
@@ -1133,12 +1121,16 @@ impl<B: Backend> TokenDecoderRunner<B> {
src_slot,
},
)?;
- let copy_size =
+ let byte_len =
count.checked_mul(std::mem::size_of::<u32>()).ok_or(TtsModelConfigError::AsyncChainCopySizeOverflow {
count,
})?;
- let results_buffer = self.ctx.async_chain_results.borrow();
- encoder.encode_copy(&results_buffer, src_offset..src_offset + copy_size, dst.0, dst.1..dst.1 + copy_size);
+ encoder.encode_copy(
+ &self.ctx.async_chain_results,
+ src_offset..src_offset + byte_len,
+ dst,
+ dst_offset..dst_offset + byte_len,
+ );
Ok(())
}
@@ -1146,8 +1138,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
&self,
slot: usize,
) -> Result<u32, Error> {
- let results_buffer = self.ctx.async_chain_results.borrow();
- let ptr = results_buffer.cpu_ptr().as_ptr() as *const u32;
let capacity = self.ctx.async_chain_capacity;
if slot >= capacity {
return Err(TtsModelConfigError::AsyncChainResultSlotOutOfBounds {
@@ -1156,7 +1146,12 @@ impl<B: Backend> TokenDecoderRunner<B> {
}
.into());
}
- Ok(unsafe { *ptr.add(slot) })
+ let buffer_range = self.ctx.async_chain_results.as_buffer_range_ref();
+ let range = buffer_range.range();
+ Ok(unsafe {
+ *((buffer_range.buffer().cpu_ptr().as_ptr() as *const u8)
+ .add(range.start + slot * std::mem::size_of::<u32>()) as *const u32)
+ })
}
pub(super) fn take_instrumentation(&mut self) -> RunnerInstrumentation {
@@ -1168,9 +1163,9 @@ impl<B: Backend> TokenDecoderRunner<B> {
}
}
-fn read_sampled_token_from_sampling_output<B: Backend>(state: &ForwardPassState<B>) -> Result<u64, Error> {
- let output = state.sampling_output().ok_or(Error::SamplingFailed)?;
- let tokens = output.as_slice::<u32>();
- let token = tokens.first().copied().ok_or(Error::SamplingFailed)?;
+fn read_sampled_token_from_sampling_output<B: Backend>(sampling_output: &Allocation<B>) -> Result<u64, Error> {
+ let buffer_range = sampling_output.as_buffer_range_ref();
+ let range = buffer_range.range();
+ let token = unsafe { *((buffer_range.buffer().cpu_ptr().as_ptr() as *const u8).add(range.start) as *const u32) };
Ok(u64::from(token))
}
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_support.rs b/crates/backend-uzu/src/session/tts_session/decoder_support.rs
index 8b708f6da..68cdb2653 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_support.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_support.rs
@@ -1,7 +1,7 @@
use super::*;
pub(super) type PreInjectionEncodeCallback<'a, B> =
- dyn FnMut(&TokenDecoderRunner<B>, &ForwardPassState<B>, &mut Encoder<B>) -> Result<(), Error> + 'a;
+ dyn FnMut(&mut TokenDecoderRunner<B>, &mut Encoder<B>) -> Result<(), Error> + 'a;
pub(super) enum EmbeddingInjection {
None,
@@ -362,7 +362,7 @@ impl<'a, F: FnMut(&AudioPcmBatch)> StreamingSynthesisState<'a, F> {
let ready_frames = pending.ready_frames;
let next_chunk_frames = pending.next_chunk_frames;
let submission_decode_duration = pending.submission_decode_duration;
- let _ = self.pending_chunk.take().ok_or(Error::GenerateFailed)?;
+ self.pending_chunk.take().ok_or(Error::GenerateFailed)?;
(
step_stats,
ready_frames,
diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
index 0e4cbd902..ef1b5dc55 100644
--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs
@@ -4,8 +4,11 @@ use regex::Regex;
use super::*;
use crate::{
- array::Array,
+ array::ArrayContextExt,
+ backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},
config::TtsMessageProcessorConfig,
+ encodable_block::SamplingInputs,
+ forward_pass::token_inputs::TokenInputs,
session::types::{TtsModelConfigError, TtsPromptConfigError},
};
@@ -35,10 +38,13 @@ pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {
struct FishAudioSemanticBridge<B: Backend> {
embedding_rows_sum: <B::Kernels as Kernels>::EmbeddingRowsSumKernel,
projection: <B::Kernels as ManualKernels>::MatmulKernel,
- tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,
- codebook_embeddings: Array<B>,
- codebook_token_indices: Array<B>,
- projection_weights: Option<Array<B>>,
+ codebook_embeddings: Allocation<B>,
+ codebook_token_indices: Allocation<B>,
+ projection_weights: Option<Allocation<B>>,
+ num_codebooks: usize,
+ codebook_size: usize,
+ slow_model_dim: usize,
+ fast_model_dim: usize,
}
impl<B: Backend> FishAudioSemanticBridge<B> {
@@ -48,7 +54,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {
config: &crate::config::FishAudioTextDecoderConfig,
data_type: DataType,
) -> Result<Self, Error> {
- let codebook_embeddings = load_float_tensor_array(
+ let codebook_embeddings = load_float_tensor_allocation(
parameter_tree,
"text_decoder.codebook_embeddings.weights",
[
@@ -58,7 +64,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {
data_type,
)?;
let fast_model_projection = if config.fast_model_projection_config.is_some() {
- Some(load_float_tensor_array(
+ Some(load_float_tensor_allocation(
parameter_tree,
"text_decoder.fast_model_projection.weights",
[config.fast_model_dim, config.slow_model_dim],
@@ -83,64 +89,52 @@ impl<B: Backend> FishAudioSemanticBridge<B> {
fn new(
context: &B::Context,
data_type: DataType,
- codebook_embeddings: Array<B>,
- fast_model_projection: Option<Array<B>>,
+ codebook_embeddings: Allocation<B>,
+ fast_model_projection: Option<Allocation<B>>,
num_codebooks: usize,
codebook_size: usize,
slow_model_dim: usize,
fast_model_dim: usize,
) -> Result<Self, Error> {
- if codebook_embeddings.shape()
- != [num_codebooks.checked_mul(codebook_size).ok_or(Error::UnableToLoadConfig)?, slow_model_dim]
- || codebook_embeddings.data_type() != data_type
- {
- return Err(Error::UnableToLoadConfig);
- }
- if let Some(ref projection) = fast_model_projection
- && (projection.shape() != [fast_model_dim, slow_model_dim] || projection.data_type() != data_type)
- {
- return Err(Error::UnableToLoadConfig);
- }
-
let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type)
.map_err(unable_to_create_context)?;
let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
.map_err(unable_to_create_context)?;
- let tensor_copy =
- <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;
- let codebook_token_indices =
- context.create_array_zeros(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");
+ let codebook_token_indices = context.create_array_zeros(&[num_codebooks], DataType::U32).into_allocation();
Ok(Self {
embedding_rows_sum,
projection,
- tensor_copy,
codebook_embeddings,
codebook_token_indices,
projection_weights: fast_model_projection,
+ num_codebooks,
+ codebook_size,
+ slow_model_dim,
+ fast_model_dim,
})
}
}
-fn load_float_tensor_array<B: Backend>(
+fn load_float_tensor_allocation<B: Backend>(
parameter_tree: &crate::parameters::ParameterTree<B::Context>,
key: &str,
expected_shape: [usize; 2],
target_data_type: DataType,
-) -> Result<Array<B>, Error> {
- let array = parameter_tree.leaf_array(key).map_err(|_| Error::UnableToLoadWeights)?;
- if array.shape() != expected_shape {
+) -> Result<Allocation<B>, Error> {
+ let leaf = parameter_tree.leaf(key).map_err(|_| Error::UnableToLoadWeights)?;
+ if leaf.shape() != expected_shape {
return Err(Error::UnableToLoadConfig);
}
- if array.data_type() != target_data_type {
+ if leaf.data_type() != target_data_type {
return Err(TtsModelConfigError::FishAudioTensorDataTypeMismatch {
key: key.into(),
expected: target_data_type,
- actual: array.data_type(),
+ actual: leaf.data_type(),
}
.into());
}
- Ok(array)
+ leaf.read_allocation().map_err(|_| Error::UnableToLoadWeights)
}
fn validate_fishaudio_decoder_contract(
@@ -303,8 +297,8 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
runtime_config,
)?;
- let activation_data_type = slow_runner.single_hidden_capture.data_type();
- let fast_data_type = fast_runner.single_override_embedding.data_type();
+ let activation_data_type = slow_runner.activation_data_type();
+ let fast_data_type = fast_runner.activation_data_type();
if fast_data_type != activation_data_type {
return Err(Error::UnableToLoadConfig);
}
@@ -510,42 +504,25 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
}
fn encode_project_slow_hidden_to_fast_on(
- context: &B::Context,
semantic_bridge: &mut FishAudioSemanticBridge<B>,
- slow_hidden_capture: &Array<B>,
- output_embedding: &Array<B>,
+ slow_hidden_capture: &Allocation<B>,
+ output_embedding: &mut Allocation<B>,
slow_model_dim: usize,
fast_model_dim: usize,
encoder: &mut Encoder<B>,
) -> Result<(), Error> {
- let model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;
-
if let Some(weights) = semantic_bridge.projection_weights.as_ref() {
- let hidden = slow_hidden_capture;
- let weights = weights;
- let output = output_embedding;
- if hidden.shape() != [1, slow_model_dim]
- || output.shape() != [1, fast_model_dim]
- || weights.shape() != [fast_model_dim, slow_model_dim]
- {
+ if semantic_bridge.fast_model_dim != fast_model_dim || semantic_bridge.slow_model_dim != slow_model_dim {
return Err(Error::GenerateFailed);
}
-
- let hidden_buffer = hidden.buffer();
- let weights_buffer = weights.buffer();
- let output_buffer = output.buffer();
- let hidden_buffer = hidden_buffer.borrow();
- let weights_buffer = weights_buffer.borrow();
- let mut output_buffer = output_buffer.borrow_mut();
semantic_bridge.projection.encode(
- context,
MatmulArguments {
- a: &hidden_buffer,
- a_offset: hidden.offset() as u64,
- b: &weights_buffer,
+ a: slow_hidden_capture,
+ a_offset: 0,
+ b: weights,
ab_scale: 1.0,
c: MatmulArgumentC::None,
- d: &mut output_buffer,
+ d: output_embedding,
batch_dim: 1,
input_dim: slow_model_dim as u32,
output_dim: fast_model_dim as u32,
@@ -559,21 +536,11 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
return Err(Error::UnableToLoadConfig);
}
- let hidden = slow_hidden_capture;
- let output = output_embedding;
- if hidden.shape() != [1, slow_model_dim] || output.shape() != [1, fast_model_dim] {
- return Err(Error::GenerateFailed);
- }
- let hidden_buffer = hidden.buffer();
- let hidden_buffer = hidden_buffer.borrow();
- let output_buffer = output.buffer();
- let mut output_buffer = output_buffer.borrow_mut();
- semantic_bridge.tensor_copy.encode(
- (&*hidden_buffer, hidden.offset()),
- (&mut *output_buffer, output.offset()),
- model_dim_u32,
- encoder,
+ debug_assert_eq!(
+ slow_hidden_capture.as_buffer_range_ref().range().len(),
+ output_embedding.as_buffer_range_ref().range().len()
);
+ encoder.encode_copy(slow_hidden_capture, .., output_embedding, ..);
Ok(())
}
@@ -585,9 +552,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
/// `gpu_tokens_buffer` at the given byte offset.
fn encode_slow_codebook_sum_from_gpu_tokens_on(
semantic_bridge: &mut FishAudioSemanticBridge<B>,
- slow_sum_embedding: &Array<B>,
+ slow_sum_embedding: &mut Allocation<B>,
first_code: u32,
- source_runner: &TokenDecoderRunner<B>,
+ source_runner: &mut TokenDecoderRunner<B>,
source_slot: usize,
num_codebooks: usize,
codebook_size: usize,
@@ -600,24 +567,26 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
// Write first_code at position 0 from CPU.
{
- let token_indices = &mut semantic_bridge.codebook_token_indices;
- if token_indices.shape() != [num_codebooks] || token_indices.data_type() != DataType::U32 {
+ if semantic_bridge.num_codebooks != num_codebooks {
return Err(Error::GenerateFailed);
}
- let indices_slice = token_indices.as_slice_mut::<u32>();
- indices_slice[0] = first_code;
+ let buffer_range = semantic_bridge.codebook_token_indices.as_buffer_range_ref();
+ let range = buffer_range.range();
+ unsafe {
+ let ptr = (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start) as *mut u32;
+ *ptr = first_code;
+ }
}
// Blit residual codes from GPU: async_chain_results[0..N-2] -> codebook_token_indices[1..N-1]
let residual_count = num_codebooks.saturating_sub(1);
if residual_count > 0 {
let dst_byte_offset = std::mem::size_of::<u32>(); // skip slot 0
- let token_indices_buffer = semantic_bridge.codebook_token_indices.buffer();
- let mut token_indices_buffer = token_indices_buffer.borrow_mut();
source_runner.copy_async_chain_results_to_on(
encoder,
source_slot,
- (&mut *token_indices_buffer, semantic_bridge.codebook_token_indices.offset() + dst_byte_offset),
+ &mut semantic_bridge.codebook_token_indices,
+ dst_byte_offset,
residual_count,
)?;
}
@@ -630,25 +599,16 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
let codebook_token_indices = &semantic_bridge.codebook_token_indices;
let codebook_embeddings = &semantic_bridge.codebook_embeddings;
- let slow_sum = slow_sum_embedding;
- if codebook_token_indices.shape() != [num_codebooks]
- || codebook_embeddings.shape() != [total_vocab, slow_model_dim]
- || slow_sum.shape() != [1, slow_model_dim]
- || codebook_embeddings.data_type() != slow_sum.data_type()
+ if semantic_bridge.num_codebooks != num_codebooks
+ || semantic_bridge.codebook_size != codebook_size
+ || semantic_bridge.slow_model_dim != slow_model_dim
{
return Err(Error::GenerateFailed);
}
-
- let codebook_token_indices_buffer = codebook_token_indices.buffer();
- let codebook_token_indices_buffer = codebook_token_indices_buffer.borrow();
- let codebook_embeddings_buffer = codebook_embeddings.buffer();
- let codebook_embeddings_buffer = codebook_embeddings_buffer.borrow();
- let slow_sum_buffer = slow_sum.buffer();
- let mut slow_sum_buffer = slow_sum_buffer.borrow_mut();
semantic_bridge.embedding_rows_sum.encode(
- (&*codebook_token_indices_buffer, codebook_token_indices.offset()),
- (&*codebook_embeddings_buffer, codebook_embeddings.offset()),
- (&mut *slow_sum_buffer, slow_sum.offset()),
+ codebook_token_indices,
+ codebook_embeddings,
+ slow_sum_embedding,
num_codebooks_u32,
total_vocab_u32,
slow_model_dim_u32,
@@ -680,8 +640,19 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
let context = Rc::clone(self.fast_runner.context());
let mut shared_encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
- let total_fast_count =
- self.encode_fast_passes_on(&mut shared_encoder, first_code, fast_vocab_limit, followup_count, sampling)?;
+ let mut pending_token_inputs = Vec::new();
+ let mut pending_sampling_inputs = Vec::new();
+ let mut pending_sampling_outputs = Vec::new();
+ let total_fast_count = self.encode_fast_passes_on(
+ &mut shared_encoder,
+ first_code,
+ fast_vocab_limit,
+ followup_count,
+ sampling,
+ &mut pending_token_inputs,
+ &mut pending_sampling_inputs,
+ &mut pending_sampling_outputs,
+ )?;
self.encode_slow_pass_on(
&mut shared_encoder,
current_semantic_token,
@@ -689,8 +660,12 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
post_scale,
sampling,
by_codebook,
+ &mut pending_token_inputs,
+ &mut pending_sampling_inputs,
+ &mut pending_sampling_outputs,
)?;
- self.slow_runner.submit_and_wait_command_buffer(shared_encoder)?;
+ let submit_result = self.slow_runner.submit_and_wait_command_buffer(shared_encoder);
+ submit_result?;
for pass in 0..total_fast_count {
let sampled = self.fast_runner.read_async_chain_result(pass)?;
@@ -714,6 +689,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
fast_vocab_limit: usize,
followup_count: usize,
sampling: &mut TextSamplingState,
+ pending_token_inputs: &mut Vec<TokenInputs<B>>,
+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
+ pending_sampling_outputs: &mut Vec<Allocation<B>>,
) -> Result<usize, Error> {
let (slow_runner, fast_runner, semantic_bridge) =
(&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);
@@ -721,18 +699,16 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
let slow_model_dim = self.slow_model_dim;
let fast_model_dim = self.fast_model_dim;
- let mut pre_projection =
- |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {
- Self::encode_project_slow_hidden_to_fast_on(
- runner.context().as_ref(),
- semantic_bridge,
- slow_hidden_capture,
- &runner.single_override_embedding,
- slow_model_dim,
- fast_model_dim,
- encoder,
- )
- };
+ let mut pre_projection = |runner: &mut TokenDecoderRunner<B>, encoder: &mut Encoder<B>| {
+ Self::encode_project_slow_hidden_to_fast_on(
+ semantic_bridge,
+ slow_hidden_capture,
+ &mut runner.single_override_embedding,
+ slow_model_dim,
+ fast_model_dim,
+ encoder,
+ )
+ };
fast_runner.encode_first_and_followup_tokens_on(
encoder,
@@ -742,6 +718,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
followup_count,
Some(fast_vocab_limit),
sampling,
+ pending_token_inputs,
+ pending_sampling_inputs,
+ pending_sampling_outputs,
)
}
@@ -754,6 +733,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
post_scale: Option<f32>,
sampling: &mut TextSamplingState,
by_codebook: &[Vec<u32>],
+ pending_token_inputs: &mut Vec<TokenInputs<B>>,
+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
+ pending_sampling_outputs: &mut Vec<Allocation<B>>,
) -> Result<(), Error> {
let (slow_runner, fast_runner, semantic_bridge) =
(&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);
@@ -772,20 +754,19 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
None
};
- let mut pre_codebook_sum =
- |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {
- Self::encode_slow_codebook_sum_from_gpu_tokens_on(
- semantic_bridge,
- &runner.single_override_embedding,
- first_code,
- fast_runner,
- 0,
- num_codebooks,
- codebook_size,
- slow_model_dim,
- encoder,
- )
- };
+ let mut pre_codebook_sum = |runner: &mut TokenDecoderRunner<B>, encoder: &mut Encoder<B>| {
+ Self::encode_slow_codebook_sum_from_gpu_tokens_on(
+ semantic_bridge,
+ &mut runner.single_override_embedding,
+ first_code,
+ fast_runner,
+ 0,
+ num_codebooks,
+ codebook_size,
+ slow_model_dim,
+ encoder,
+ )
+ };
slow_runner.encode_next_step_on(
encoder,
@@ -797,6 +778,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {
slow_sampling_mask,
true, // capture_hidden
Some(&mut pre_codebook_sum),
+ pending_token_inputs,
+ pending_sampling_inputs,
+ pending_sampling_outputs,
)
}
diff --git a/crates/backend-uzu/src/speculators/ngram_speculator.rs b/crates/backend-uzu/src/speculators/ngram_speculator.rs
index 077e62174..cd8e5af60 100644
--- a/crates/backend-uzu/src/speculators/ngram_speculator.rs
+++ b/crates/backend-uzu/src/speculators/ngram_speculator.rs
@@ -182,8 +182,7 @@ impl<B: Deref<Target = [u8]> + Send + Sync> NGramSpeculator<B> {
let mut off = 0;
let max_order = u32::from_le_bytes(bytes[off..off + 4].try_into().unwrap());
- // Discount parameter is baked into the exported probabilities at training time.
- let _discount = f32::from_le_bytes(bytes[off + 4..off + 8].try_into().unwrap());
+ // Bytes [4..8] store the discount parameter, but exported probabilities are already discounted.
off += 8;
let mut tables: Vec<TaggedTableLayout> = Vec::with_capacity(max_order as usize);
diff --git a/crates/backend-uzu/src/tracer/trace_validator.rs b/crates/backend-uzu/src/tracer/trace_validator.rs
index 2b06ba8cf..43f8a3bea 100644
--- a/crates/backend-uzu/src/tracer/trace_validator.rs
+++ b/crates/backend-uzu/src/tracer/trace_validator.rs
@@ -16,18 +16,13 @@ use ndarray::{IxDyn, s};
use num_traits::NumCast;
use crate::{
- ArrayElement, DataType,
+ ArrayElement, DataType, allocation_to_vec,
array::Array,
- backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
+ backends::common::{Allocation, Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
classifier::Classifier,
config::ModelMetadata,
- encodable_block::{EncodingParameters, Sampling},
- forward_pass::{
- cache_layers::CacheLayers,
- scratch_buffers::ScratchBuffers,
- state::{ArrayId, ForwardPassState},
- traces::ActivationTrace,
- },
+ encodable_block::{DecoderDecodeInput, EncodingParameters, Sampling},
+ forward_pass::{cache_layers::CacheLayers, token_inputs::TokenInputs, traces::ActivationTrace},
language_model::{
language_model_generator_context::LanguageModelGeneratorContext,
sampler::{ArgmaxSampler, LogitsSampler},
@@ -196,7 +191,6 @@ impl<B: Backend> TraceValidator<B> {
SpeculatorConfig::default(),
SamplingSeed::default(),
AsyncBatchSize::default(),
- false,
);
let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &metadata)?;
let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());
@@ -246,39 +240,46 @@ impl<B: Backend> TraceValidator<B> {
let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");
let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");
- let token_seeds: Vec<u64> = vec![0; token_ids.len()];
-
- let mut state = ForwardPassState::new_llm(
- ctx.context.clone(),
- &ctx.decoder_config,
+ let token_inputs = TokenInputs::new_llm(
+ ctx.context.as_ref(),
&ctx.model_shape,
- &ctx.scratch_buffers,
- ctx.cache_layers.clone(),
- ctx.shared_buffers.clone(),
&token_ids,
None,
&token_positions,
None,
- &token_seeds,
- token_ids.len(),
+ None,
/*sampling_start=*/ 0,
/*sampling_length=*/ token_ids.len(),
- false,
- false,
- None,
- None,
);
+ let mut traces = ActivationTrace::new_llm(ctx.context.as_ref(), &ctx.model_shape, token_ids.len());
let mut encoder =
Encoder::<B>::new(ctx.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
-
- ctx.executables
- .encode(&mut state, &EncodingParameters::new(), &mut encoder)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ {
+ let mut cache_layers = ctx.cache_layers.borrow_mut();
+ let decoder_arguments = token_inputs.decoder_arguments(
+ &ctx.model_shape,
+ ctx.shared_buffers.as_ref(),
+ Some(&mut *cache_layers),
+ token_ids.len(),
+ /*sampling_start=*/ 0,
+ /*sampling_length=*/ token_ids.len(),
+ #[cfg(feature = "tracing")]
+ Some(&mut traces),
+ );
+ ctx.executables
+ .encode_decode(
+ decoder_arguments,
+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),
+ None,
+ &EncodingParameters::new(),
+ &mut encoder,
+ )
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ }
let pending = encoder.end_encoding().submit();
pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
- let traces = state.traces().clone();
let data_type = ctx.model_shape.activation_data_type();
// Common layer validation
@@ -286,38 +287,50 @@ impl<B: Backend> TraceValidator<B> {
// LLM-specific: KV cache validation
let transformer_layers: Vec<usize> = {
- let cache = state.cache_layers().unwrap().borrow();
+ let cache = ctx.cache_layers.borrow();
cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_transformer().map(|_| index)).collect()
};
for index in transformer_layers {
- let keys = state.array(ArrayId::Keys(index));
- let values = state.array(ArrayId::Values(index));
+ let cache = ctx.cache_layers.borrow();
+ let kv = cache.data[index].as_transformer().expect("Expected transformer layer");
if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.keys", index),
- metrics: Self::validate_array(data_type, &expected, &keys, Some(ArrayTransform::KVCacheSlice)),
+ metrics: Self::validate_allocation(
+ data_type,
+ &expected,
+ &kv.keys,
+ &kv.shape,
+ Some(ArrayTransform::KVCacheSlice),
+ ),
});
}
if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.values", index),
- metrics: Self::validate_array(data_type, &expected, &values, Some(ArrayTransform::KVCacheSlice)),
+ metrics: Self::validate_allocation(
+ data_type,
+ &expected,
+ &kv.values,
+ &kv.shape,
+ Some(ArrayTransform::KVCacheSlice),
+ ),
});
}
}
// LLM-specific: SSM state validation
let ssm_layers: Vec<usize> = {
- let cache = state.cache_layers().unwrap().borrow();
+ let cache = ctx.cache_layers.borrow();
cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_state_space().map(|_| index)).collect()
};
for index in ssm_layers {
- let conv_state = state.array(ArrayId::SsmConvState(index));
- let ssm_state = state.array(ArrayId::SsmState(index));
+ let cache = ctx.cache_layers.borrow();
+ let ssm = cache.data[index].as_state_space().expect("Expected SSM layer");
for path in [
format!("updated_state.{}.conv_state", index),
@@ -326,10 +339,11 @@ impl<B: Backend> TraceValidator<B> {
if let Ok(expected) = traces_view.leaf_array(&path) {
results.push(TracerValidationResult {
name: path,
- metrics: Self::validate_array(
+ metrics: Self::validate_optional_allocation(
data_type,
&expected,
- &conv_state,
+ ssm.conv_state.as_ref(),
+ &ssm.conv_shape,
Some(ArrayTransform::SsmConvState),
),
});
@@ -343,7 +357,7 @@ impl<B: Backend> TraceValidator<B> {
if let Ok(expected) = traces_view.leaf_array(&path) {
results.push(TracerValidationResult {
name: path,
- metrics: Self::validate_array(data_type, &expected, &ssm_state, None),
+ metrics: Self::validate_allocation(data_type, &expected, &ssm.ssm_state, &ssm.ssm_shape, None),
});
}
}
@@ -351,13 +365,13 @@ impl<B: Backend> TraceValidator<B> {
// LLM-specific: DeltaNet state validation
let delta_net_layers: Vec<usize> = {
- let cache = state.cache_layers().unwrap().borrow();
+ let cache = ctx.cache_layers.borrow();
cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_delta_net().map(|_| index)).collect()
};
for index in delta_net_layers {
- let conv_state = state.array(ArrayId::DeltaNetConvState(index));
- let ssm_state = state.array(ArrayId::DeltaNetSsmState(index));
+ let cache = ctx.cache_layers.borrow();
+ let delta = cache.data[index].as_delta_net().expect("Expected DeltaNet layer");
for path in [
format!("updated_state.{}.conv_state", index),
@@ -366,10 +380,11 @@ impl<B: Backend> TraceValidator<B> {
if let Ok(expected) = traces_view.leaf_array(&path) {
results.push(TracerValidationResult {
name: path,
- metrics: Self::validate_array(
+ metrics: Self::validate_allocation(
data_type,
&expected,
- &conv_state,
+ &delta.conv_state,
+ &delta.conv_shape,
Some(ArrayTransform::SsmConvState),
),
});
@@ -383,7 +398,13 @@ impl<B: Backend> TraceValidator<B> {
if let Ok(expected) = traces_view.leaf_array(&path) {
results.push(TracerValidationResult {
name: path,
- metrics: Self::validate_array(data_type, &expected, &ssm_state, None),
+ metrics: Self::validate_allocation(
+ data_type,
+ &expected,
+ &delta.ssm_state,
+ &delta.ssm_shape,
+ None,
+ ),
});
}
}
@@ -392,7 +413,7 @@ impl<B: Backend> TraceValidator<B> {
// LLM-specific: Token comparison
let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {
let expected_tokens = Self::get_tokens_from_logits(&expected_logits);
- let produced_tokens = Self::get_tokens_from_logits(&traces.borrow().logits);
+ let produced_tokens = Self::get_tokens_from_logits(&traces.logits);
expected_tokens
.iter()
.zip(produced_tokens.iter())
@@ -506,68 +527,68 @@ impl<B: Backend> TraceValidator<B> {
// ========================================================================
fn validate_layer_traces(
- traces: &Rc<RefCell<ActivationTrace<B>>>,
+ traces: &ActivationTrace<B>,
traces_view: &ParameterTree<B::Context>,
data_type: DataType,
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
let validate = |path: &str, array: &Array<B>| -> Option<TracerValidationResult> {
- if traces_view.leaf_array(path).is_ok() {
+ if let Ok(expected) = traces_view.leaf_array(path) {
Some(TracerValidationResult {
name: path.to_string(),
- metrics: Self::validate_array_with_name(data_type, traces_view, path, array),
+ metrics: Self::validate_allocation(data_type, &expected, array.allocation(), array.shape(), None),
})
} else {
None
}
};
- for (index, layer_traces) in traces.borrow().layer_results.iter().enumerate() {
+ for (index, layer_traces) in traces.layer_results.iter().enumerate() {
let path = |suffix: &str| -> String {
format!("activation_trace.layer_results.{}.activation_trace.{}", index, suffix)
};
- if let Some(r) = validate(&path("inputs"), &layer_traces.borrow().inputs) {
+ if let Some(r) = validate(&path("inputs"), &layer_traces.inputs) {
results.push(r);
}
- if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.borrow().pre_attention_norm) {
+ if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.pre_attention_norm) {
results.push(r);
}
- if let Some(r) = validate(&path("mixer"), &layer_traces.borrow().attention) {
+ if let Some(r) = validate(&path("mixer"), &layer_traces.attention) {
results.push(r);
}
- if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.borrow().post_attention_norm) {
+ if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.post_attention_norm) {
results.push(r);
}
- if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.borrow().mlp_inputs) {
+ if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.mlp_inputs) {
results.push(r);
}
- if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.borrow().pre_mlp_norm) {
+ if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.pre_mlp_norm) {
results.push(r);
}
- if let Some(r) = validate(&path("mlp"), &layer_traces.borrow().mlp) {
+ if let Some(r) = validate(&path("mlp"), &layer_traces.mlp) {
results.push(r);
}
- if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.borrow().post_mlp_norm) {
+ if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.post_mlp_norm) {
results.push(r);
}
let outputs_path = format!("activation_trace.layer_results.{}.outputs", index);
- if let Some(r) = validate(&outputs_path, &layer_traces.borrow().outputs) {
+ if let Some(r) = validate(&outputs_path, &layer_traces.outputs) {
results.push(r);
}
}
// Output norm (common to all models)
- if let Some(r) = validate("activation_trace.output_norm", &traces.borrow().output_norm) {
+ if let Some(r) = validate("activation_trace.output_norm", &traces.output_norm) {
results.push(r);
}
// Logits (common to all models, but path may vary)
- if let Some(r) = validate("activation_trace.logits", &traces.borrow().logits) {
+ if let Some(r) = validate("activation_trace.logits", &traces.logits) {
results.push(r);
- } else if let Some(r) = validate("logits", &traces.borrow().logits) {
+ } else if let Some(r) = validate("logits", &traces.logits) {
results.push(r);
}
@@ -575,22 +596,23 @@ impl<B: Backend> TraceValidator<B> {
}
fn validate_classifier_traces(
- traces: &Rc<RefCell<ActivationTrace<B>>>,
+ traces: &ActivationTrace<B>,
traces_view: &ParameterTree<B::Context>,
data_type: DataType,
) -> Vec<TracerValidationResult> {
let mut results = Vec::new();
// Output pooling (classifier-specific)
- if let Some(output_pooling) = &traces.borrow().output_pooling {
- if traces_view.leaf_array("activation_trace.output_pooling").is_ok() {
+ if let Some(output_pooling) = &traces.output_pooling {
+ if let Ok(expected) = traces_view.leaf_array("activation_trace.output_pooling") {
results.push(TracerValidationResult {
name: "activation_trace.output_pooling".to_string(),
- metrics: Self::validate_array_with_name(
+ metrics: Self::validate_allocation(
data_type,
- traces_view,
- "activation_trace.output_pooling",
- output_pooling,
+ &expected,
+ output_pooling.allocation(),
+ output_pooling.shape(),
+ None,
),
});
}
@@ -600,40 +622,78 @@ impl<B: Backend> TraceValidator<B> {
}
// ========================================================================
- // Array Validation
+ // Allocation Validation
// ========================================================================
- fn validate_array(
+ fn validate_allocation(
data_type: DataType,
expected_array: &Array<B>,
- produced_array: &Array<B>,
+ produced_allocation: &Allocation<B>,
+ produced_shape: &[usize],
transform: Option<ArrayTransform>,
) -> TracerValidationMetrics {
match data_type {
- DataType::F16 => Self::validate_array_of_type::<f16>(expected_array, produced_array, transform),
- DataType::BF16 => Self::validate_array_of_type::<bf16>(expected_array, produced_array, transform),
- DataType::F32 => Self::validate_array_of_type::<f32>(expected_array, produced_array, transform),
+ DataType::F16 => {
+ Self::validate_allocation_of_type::<f16>(expected_array, produced_allocation, produced_shape, transform)
+ },
+ DataType::BF16 => Self::validate_allocation_of_type::<bf16>(
+ expected_array,
+ produced_allocation,
+ produced_shape,
+ transform,
+ ),
+ DataType::F32 => {
+ Self::validate_allocation_of_type::<f32>(expected_array, produced_allocation, produced_shape, transform)
+ },
_ => panic!("Unsupported data type: {:?}", data_type),
}
}
- fn validate_array_with_name(
+ fn validate_optional_allocation(
data_type: DataType,
- traces_view: &ParameterTree<B::Context>,
- expected_array_path: &str,
- produced_array: &Array<B>,
+ expected_array: &Array<B>,
+ produced_allocation: Option<&Allocation<B>>,
+ produced_shape: &[usize],
+ transform: Option<ArrayTransform>,
+ ) -> TracerValidationMetrics {
+ match produced_allocation {
+ Some(produced_allocation) => {
+ Self::validate_allocation(data_type, expected_array, produced_allocation, produced_shape, transform)
+ },
+ None => match data_type {
+ DataType::F16 => {
+ Self::validate_allocation_data_of_type::<f16>(expected_array, &[], produced_shape, transform)
+ },
+ DataType::BF16 => {
+ Self::validate_allocation_data_of_type::<bf16>(expected_array, &[], produced_shape, transform)
+ },
+ DataType::F32 => {
+ Self::validate_allocation_data_of_type::<f32>(expected_array, &[], produced_shape, transform)
+ },
+ _ => panic!("Unsupported data type: {:?}", data_type),
+ },
+ }
+ }
+
+ fn validate_allocation_of_type<Precision: ArrayElement>(
+ expected_array: &Array<B>,
+ produced_allocation: &Allocation<B>,
+ produced_shape: &[usize],
+ transform: Option<ArrayTransform>,
) -> TracerValidationMetrics {
- let expected_array = traces_view.leaf_array(expected_array_path).unwrap();
- Self::validate_array(data_type, &expected_array, produced_array, None)
+ let produced = allocation_to_vec::<B, Precision>(produced_allocation);
+ Self::validate_allocation_data_of_type(expected_array, &produced, produced_shape, transform)
}
- fn validate_array_of_type<Precision: ArrayElement>(
+ fn validate_allocation_data_of_type<Precision: ArrayElement>(
expected_array: &Array<B>,
- produced_array: &Array<B>,
+ produced_slice: &[Precision],
+ produced_shape: &[usize],
transform: Option<ArrayTransform>,
) -> TracerValidationMetrics {
let expected_view = expected_array.as_view::<Precision>();
- let produced_view = produced_array.as_view::<Precision>();
+ let produced_view = ndarray::ArrayView::from_shape(IxDyn(produced_shape), produced_slice)
+ .expect("Failed to reshape allocation");
let (mut expected_data, mut produced_data) = match transform {
Some(ArrayTransform::KVCacheSlice) => {
@@ -673,7 +733,6 @@ impl<B: Backend> TraceValidator<B> {
let expected_shape = expected_data.shape().to_vec();
let produced_shape = produced_data.shape().to_vec();
- // Handle shape mismatches with leading dimension of 1
if expected_shape != produced_shape {
if expected_shape.len() == produced_shape.len() + 1
&& expected_shape.get(0) == Some(&1)
@@ -699,7 +758,6 @@ impl<B: Backend> TraceValidator<B> {
}
let reference: Vec<f32> = expected_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();
-
let result: Vec<f32> = produced_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();
let (atol, rtol, allowed_voilations_tol) = match expected_array.data_type() {
@@ -728,6 +786,28 @@ impl<B: Backend> TraceValidator<B> {
fraction_of_allowed_violations: f32,
) -> TracerValidationMetrics {
assert_eq!(result.len(), reference.len());
+ if reference.is_empty() {
+ return TracerValidationMetrics {
+ atol,
+ rtol,
+ fraction_of_allowed_violations,
+ reference_shape,
+ result_shape,
+ num_violations: 0,
+ max_allowed_violations: 0,
+ max_err_idx: 0,
+ max_err: 0.0,
+ max_err_rel: 0.0,
+ max_err_reference_value: 0.0,
+ rms_diff: 0.0,
+ rms_result: 0.0,
+ rms_reference: 0.0,
+ rel_rms_reference: 0.0,
+ diff_max: 0.0,
+ diff_avg: 0.0,
+ result_nan: false,
+ };
+ }
let mut num_violations = 0;
let mut max_err = 0.0f32;
@@ -848,10 +928,6 @@ impl<B: Backend> TraceValidator<B> {
return;
}
- let decoder_config = &context.decoder_config;
- context.scratch_buffers =
- ScratchBuffers::new(context.context.as_ref(), decoder_config, &context.model_shape, desired_suffix_length);
-
context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(
context.context.as_ref(),
&context.model_shape,
@@ -859,6 +935,7 @@ impl<B: Backend> TraceValidator<B> {
desired_suffix_length,
)));
+ let decoder_config = context.model_config.decoder_config().expect("Failed to resolve decoder config");
let intermediate_dtype: DataType = decoder_config.output_norm_config.scale_precision.into();
context.kv_cache_update = Box::new(
@@ -866,13 +943,8 @@ impl<B: Backend> TraceValidator<B> {
.expect("Failed to create KV cache update kernel"),
);
- context.gpu_sampler = Sampling::new(
- context.context.as_ref(),
- intermediate_dtype,
- desired_suffix_length,
- decoder_config.vocab_size,
- )
- .expect("Failed to create sampling kernel");
+ context.gpu_sampler =
+ Sampling::new(context.context.as_ref(), intermediate_dtype).expect("Failed to create sampling kernel");
}
fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {
diff --git a/crates/backend-uzu/tests/common/helpers.rs b/crates/backend-uzu/tests/common/helpers.rs
index 8c3a79ccd..e30b9a93a 100644
--- a/crates/backend-uzu/tests/common/helpers.rs
+++ b/crates/backend-uzu/tests/common/helpers.rs
@@ -1,32 +1,58 @@
-use std::rc::Rc;
+use std::{mem::size_of, rc::Rc};
-use backend_uzu::backends::common::{Backend, Buffer, Context, DenseBuffer};
+use backend_uzu::{
+ ArrayElement, allocation_copy_from_slice,
+ backends::common::{Allocation, AllocationType, Backend, Context, Encoder},
+};
-#[allow(dead_code)]
-pub fn alloc_buffer<B: Backend, T>(
+pub fn allocation_size_bytes<T>(elements_count: usize) -> usize {
+ elements_count * size_of::<T>()
+}
+
+pub fn alloc_allocation<B: Backend, T>(
context: &B::Context,
elements_count: usize,
-) -> B::DenseBuffer {
- context.create_buffer(elements_count * size_of::<T>()).expect("Failed to create buffer")
+) -> Allocation<B> {
+ context
+ .create_allocation(allocation_size_bytes::<T>(elements_count), AllocationType::Global)
+ .expect("Failed to create allocation")
}
-pub fn alloc_buffer_with_data<B: Backend, T: bytemuck::NoUninit>(
+pub fn alloc_allocation_with_data<B: Backend, T: ArrayElement>(
context: &B::Context,
data: &[T],
-) -> B::DenseBuffer {
- if data.len() == 0 {
- // Metal doesn't allow creating 0-byte buffers, create a minimal buffer instead
- return context.create_buffer(1).expect("Failed to create buffer");
- }
+) -> Allocation<B> {
+ let mut allocation = context
+ .create_allocation(allocation_size_bytes::<T>(data.len()), AllocationType::Global)
+ .expect("Failed to create allocation");
+ allocation_copy_from_slice(&mut allocation, data).expect("Failed to initialize allocation");
+ allocation
+}
- let slice: &[u8] = bytemuck::cast_slice(data);
- let buffer = context.create_buffer(slice.len()).expect("Failed to create buffer");
- let bytes = unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, buffer.size()) };
- bytes.copy_from_slice(slice);
- buffer
+pub fn allocation_to_vec<B: Backend, T: ArrayElement>(allocation: &Allocation<B>) -> Vec<T> {
+ backend_uzu::allocation_to_vec(allocation)
+}
+
+pub fn allocation_prefix_to_vec<B: Backend, T: ArrayElement>(
+ allocation: &Allocation<B>,
+ elements_count: usize,
+) -> Vec<T> {
+ let mut values = allocation_to_vec::<B, T>(allocation);
+ values.truncate(elements_count);
+ values
+}
+
+pub fn write_allocation<B: Backend, T: ArrayElement>(
+ allocation: &mut Allocation<B>,
+ data: &[T],
+) {
+ allocation_copy_from_slice(allocation, data).expect("Failed to write allocation")
}
-#[allow(dead_code)]
pub fn create_context<B: Backend>() -> Rc<<B as Backend>::Context> {
B::Context::new().expect(format!("Failed to create context for {}", std::any::type_name::<B>()).as_str())
}
+
+pub fn submit_encoder<B: Backend>(encoder: Encoder<B>) {
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+}
diff --git a/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rs b/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rs
index 75b90c04d..793085542 100644
--- a/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rs
+++ b/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rs
@@ -1,9 +1,15 @@
#![cfg(all(metal_backend, grammar_xgrammar))]
-use std::{cell::RefCell, path::PathBuf, rc::Rc};
+use std::path::PathBuf;
-use backend_uzu::{Array, DataType, backends::metal::Metal};
-use metal::{MTLDevice, MTLDeviceExt, MTLResourceOptions};
+use backend_uzu::{
+ Array, DataType,
+ backends::{
+ common::{AllocationType, Backend, Context},
+ metal::Metal,
+ },
+};
+use metal::MTLDevice;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use tokenizers::Tokenizer;
@@ -61,8 +67,10 @@ fn person_schema_metal_bitmask() {
// xgrammar expects an int32 mask tensor of shape [buffer_size]
let elems = batch * buffer_size;
let bytes = elems * core::mem::size_of::<i32>();
- let buffer = device.new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");
- let metal_bitmask = unsafe { Array::<Metal>::from_parts(Rc::new(RefCell::new(buffer)), 0, &shape, DataType::I32) };
+ let context = <Metal as Backend>::Context::new().expect("Metal context");
+ let allocation =
+ context.create_allocation(bytes.max(1), AllocationType::Global).expect("Failed to create allocation");
+ let metal_bitmask = unsafe { Array::<Metal>::from_allocation(allocation, 0, &shape, DataType::I32) };
let mut shape_i64 = [buffer_size as i64];
let mut bitmask_tensor = DLTensor {
diff --git a/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rs
index 8f5fcc270..1cbbd7302 100644
--- a/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rs
+++ b/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rs
@@ -26,7 +26,6 @@ fn build_decoding_config() -> DecodingConfig {
SpeculatorConfig::default(),
SamplingSeed::Custom(42),
AsyncBatchSize::default(),
- true,
)
}
@@ -55,6 +54,25 @@ fn request(
.to_string()
}
+fn is_unknown_answer(response: &str) -> bool {
+ let response = response.to_lowercase();
+ [
+ "not sure",
+ "not able",
+ "not aware",
+ "don't know",
+ "do not know",
+ "can't tell",
+ "cannot tell",
+ "no information",
+ "any information",
+ "more context",
+ "more details",
+ ]
+ .iter()
+ .any(|pattern| response.contains(pattern))
+}
+
#[tag(heavy)]
#[test]
fn test_context_mode_none() {
@@ -69,7 +87,8 @@ fn test_context_mode_none() {
println!("response_1: {}", response_1);
println!("response_2: {}", response_2);
- assert_eq!(response_1, response_2);
+ assert!(is_unknown_answer(&response_1));
+ assert!(is_unknown_answer(&response_2));
}
#[tag(heavy)]
diff --git a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs
index f481101f7..94648abe6 100644
--- a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs
+++ b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs
@@ -7,7 +7,7 @@ use backend_uzu::{
session::{
Session,
config::{DecodingConfig, RunConfig},
- parameter::SamplingSeed,
+ parameter::{AsyncBatchSize, SamplingSeed},
types::{Input, Message, Output},
},
};
@@ -31,6 +31,12 @@ fn test_text_session_base() {
run(build_default_text(), build_decoding_config(), 128);
}
+#[tag(heavy)]
+#[test]
+fn test_text_session_base_async_batch_1() {
+ run(build_default_text(), build_decoding_config().with_async_batch_size(AsyncBatchSize::Custom(1)), 128);
+}
+
#[tag(heavy)]
#[test]
fn test_text_session_with_prompt_lookup_speculator() {
diff --git a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs
index 526a55681..9484154b5 100644
--- a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs
+++ b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs
@@ -46,7 +46,7 @@ fn make_lengths(
batch_size: usize,
value: i32,
) -> Array<Metal> {
- context.create_array_from(&[batch_size], &vec![value; batch_size], "lengths")
+ context.create_array_from(&[batch_size], &vec![value; batch_size])
}
// ==========================================================================
@@ -121,21 +121,16 @@ fn audio_kernel_perf() {
let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();
- let tokens =
- context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");
+ let tokens = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32);
let lengths = make_lengths(&context, batch_size, frames as i32);
- let semantic_codebook = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");
- let semantic_out_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");
- let semantic_out_bias = context.create_array_zeros(&[INPUT_DIM], dt, "perf_sem_bias");
- let residual_codebooks = context.create_array_zeros(
- &[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM],
- dt,
- "perf_res_cbs",
- );
- let residual_out_proj =
- context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");
- let residual_out_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");
- let output = context.create_array_zeros(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");
+ let semantic_codebook = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt);
+ let semantic_out_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt);
+ let semantic_out_bias = context.create_array_zeros(&[INPUT_DIM], dt);
+ let residual_codebooks =
+ context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt);
+ let residual_out_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt);
+ let residual_out_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt);
+ let mut output = context.create_array_zeros(&[batch_size * frames * INPUT_DIM], dt).into_allocation();
let label = format!(
"QuantizerDecode [B={}, T={}, K={}, dim={}, cdim={}]",
@@ -144,34 +139,25 @@ fn audio_kernel_perf() {
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let tok = tokens.buffer();
- let tok = tok.borrow();
- let len = lengths.buffer();
- let len = len.borrow();
- let scb = semantic_codebook.buffer();
- let scb = scb.borrow();
- let sp = semantic_out_proj.buffer();
- let sp = sp.borrow();
- let sb = semantic_out_bias.buffer();
- let sb = sb.borrow();
- let rcb = residual_codebooks.buffer();
- let rcb = rcb.borrow();
- let rp = residual_out_proj.buffer();
- let rp = rp.borrow();
- let rb = residual_out_bias.buffer();
- let rb = rb.borrow();
- let out = output.buffer();
- let mut out = out.borrow_mut();
+ let tok = tokens.allocation();
+ let len = lengths.allocation();
+ let scb = semantic_codebook.allocation();
+ let sp = semantic_out_proj.allocation();
+ let sb = semantic_out_bias.allocation();
+ let rcb = residual_codebooks.allocation();
+ let rp = residual_out_proj.allocation();
+ let rb = residual_out_bias.allocation();
+ let out = &mut output;
kernel.encode(
- &*tok,
- &*len,
- &*scb,
- &*sp,
- &*sb,
- &*rcb,
- &*rp,
- &*rb,
- &mut *out,
+ tok,
+ len,
+ scb,
+ sp,
+ sb,
+ rcb,
+ rp,
+ rb,
+ out,
batch_size as i32,
TOTAL_CODEBOOKS as i32,
frames as i32,
@@ -208,32 +194,27 @@ fn audio_kernel_perf() {
let seq_in = frames;
let seq_out = seq_in * stride;
let ksize = stride * 2;
- let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");
- let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");
- let bias = context.create_array_zeros(&[cout], dt, "tconv_b");
- let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");
+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt);
+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt);
+ let bias = context.create_array_zeros(&[cout], dt);
+ let mut output = context.create_array_zeros(&[batch_size * cout * seq_out], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, seq_out as i32);
let label = format!("TransConv {tag} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
kernel.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
cin as i32,
cout as i32,
seq_in as i32,
@@ -261,32 +242,27 @@ fn audio_kernel_perf() {
let seq_in = frames;
let seq_out = seq_in * stride;
let ksize = stride * 2;
- let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");
- let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");
- let bias = context.create_array_zeros(&[cout], dt, "tconv_b");
- let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");
+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt);
+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt);
+ let bias = context.create_array_zeros(&[cout], dt);
+ let mut output = context.create_array_zeros(&[batch_size * cout * seq_out], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, seq_out as i32);
let label = format!("TransConv dec{block_idx} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
kernel.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
cin as i32,
cout as i32,
seq_in as i32,
@@ -321,32 +297,27 @@ fn audio_kernel_perf() {
let cin = UP_CHANNELS[2]; // 256
let cout = DECODER_DIM; // 1536
let ksize = 7;
- let input = context.create_array_zeros(&[batch_size * cin * fc_seq], dt, "fc_in");
- let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");
- let bias = context.create_array_zeros(&[cout], dt, "fc_b");
- let output = context.create_array_zeros(&[batch_size * cout * fc_seq], dt, "fc_out");
+ let input = context.create_array_zeros(&[batch_size * cin * fc_seq], dt);
+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt);
+ let bias = context.create_array_zeros(&[cout], dt);
+ let mut output = context.create_array_zeros(&[batch_size * cout * fc_seq], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, fc_seq as i32);
let label = format!("CausalConv1d first_conv [{cin}->{cout}, k={ksize}, T={fc_seq}]");
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
kernel.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
cin as i32,
cout as i32,
fc_seq as i32,
@@ -372,31 +343,26 @@ fn audio_kernel_perf() {
for &dilation in &[1, 3, 9] {
let label =
format!("CausalConv1d res dec{block_idx} [{ch}->{ch}, k={ksize}, d={dilation}, T={dec_frames}]");
- let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");
- let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");
- let bias = context.create_array_zeros(&[ch], dt, "res_b");
- let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");
+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt);
+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt);
+ let bias = context.create_array_zeros(&[ch], dt);
+ let mut output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, dec_frames as i32);
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
kernel.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
ch as i32,
ch as i32,
dec_frames as i32,
@@ -419,32 +385,27 @@ fn audio_kernel_perf() {
let cout = 1;
let ksize = 7;
let final_frames = SEMANTIC_FRAMES_10S * 2 * 2 * 8 * 8 * 4 * 2; // full resolution
- let input = context.create_array_zeros(&[batch_size * cin * final_frames], dt, "fc_in");
- let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");
- let bias = context.create_array_zeros(&[cout], dt, "fc_b");
- let output = context.create_array_zeros(&[batch_size * cout * final_frames], dt, "fc_out");
+ let input = context.create_array_zeros(&[batch_size * cin * final_frames], dt);
+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt);
+ let bias = context.create_array_zeros(&[cout], dt);
+ let mut output = context.create_array_zeros(&[batch_size * cout * final_frames], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, final_frames as i32);
let label = format!("CausalConv1d final_conv [{cin}->{cout}, k={ksize}, T={final_frames}]");
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
kernel.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
cin as i32,
cout as i32,
final_frames as i32,
@@ -477,35 +438,29 @@ fn audio_kernel_perf() {
let ksize = 7;
let label = format!("CausalConv1dResidual dec{block_idx} [{ch}->{ch}, k={ksize}, T={dec_frames}]");
- let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");
- let residual = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_res");
- let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");
- let bias = context.create_array_zeros(&[ch], dt, "res_b");
- let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");
+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt);
+ let residual = context.create_array_zeros(&[batch_size * ch * dec_frames], dt);
+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt);
+ let bias = context.create_array_zeros(&[ch], dt);
+ let mut output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, dec_frames as i32);
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let r = residual.buffer();
- let r = r.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let r = residual.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
kernel.encode(
- &*i,
- &*r,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ r,
+ w,
+ b,
+ o,
+ l,
ch as i32,
ch as i32,
dec_frames as i32,
@@ -536,31 +491,26 @@ fn audio_kernel_perf() {
let ksize = 7;
let label = format!("CausalConv1dGrouped dw up{block_idx} [{ch}, g={ch}, k={ksize}, T={frames}]");
- let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_in");
- let weight = context.create_array_zeros(&[ch * 1 * ksize], dt, "dw_w");
- let bias = context.create_array_zeros(&[ch], dt, "dw_b");
- let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_out");
+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt);
+ let weight = context.create_array_zeros(&[ch * 1 * ksize], dt);
+ let bias = context.create_array_zeros(&[ch], dt);
+ let mut output = context.create_array_zeros(&[batch_size * ch * frames], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, frames as i32);
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
kernel.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
ch as i32,
ch as i32,
frames as i32,
@@ -592,23 +542,20 @@ fn audio_kernel_perf() {
let ch_in = DEC_CHANNELS[block_idx];
{
let label = format!("HalfSnake pre-tconv dec{block_idx} [{ch_in}ch, T={dec_frames}]");
- let input = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_in");
- let alpha = context.create_array_zeros(&[ch_in], dt, "sn_a");
- let output = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_out");
+ let input = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt);
+ let alpha = context.create_array_zeros(&[ch_in], dt);
+ let mut output = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt).into_allocation();
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let a = alpha.buffer();
- let a = a.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
+ let i = input.allocation();
+ let a = alpha.allocation();
+ let o = &mut output;
kernel.encode(
- &*i,
- &*a,
- &mut *o,
+ i,
+ a,
+ o,
ch_in as i32,
dec_frames as i32,
ch_in as i32,
@@ -628,23 +575,20 @@ fn audio_kernel_perf() {
let ch_out = DEC_CHANNELS[block_idx + 1];
{
let label = format!("HalfSnake res dec{block_idx} [{ch_out}ch, T={dec_frames}]");
- let input = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_in");
- let alpha = context.create_array_zeros(&[ch_out], dt, "sn_a");
- let output = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_out");
+ let input = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt);
+ let alpha = context.create_array_zeros(&[ch_out], dt);
+ let mut output = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt).into_allocation();
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let a = alpha.buffer();
- let a = a.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
+ let i = input.allocation();
+ let a = alpha.allocation();
+ let o = &mut output;
kernel.encode(
- &*i,
- &*a,
- &mut *o,
+ i,
+ a,
+ o,
ch_out as i32,
dec_frames as i32,
ch_out as i32,
@@ -677,31 +621,26 @@ fn audio_kernel_perf() {
// Norm
{
let label = format!("NormNcs up{block_idx} [{ch}ch, T={frames}]");
- let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_in");
- let scales = context.create_array_zeros(&[ch], dt, "norm_s");
- let bias = context.create_array_zeros(&[ch], dt, "norm_b");
- let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_out");
+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt);
+ let scales = context.create_array_zeros(&[ch], dt);
+ let bias = context.create_array_zeros(&[ch], dt);
+ let mut output = context.create_array_zeros(&[batch_size * ch * frames], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, frames as i32);
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let s = scales.buffer();
- let s = s.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let s = scales.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
k_norm.encode(
- &*i,
- &*s,
- &*b,
- &mut *o,
- &*l,
+ i,
+ s,
+ b,
+ o,
+ l,
ch as i32,
frames as i32,
1e-6_f32,
@@ -718,31 +657,26 @@ fn audio_kernel_perf() {
// Pointwise conv1 (ch -> ch)
{
let label = format!("Conv1d pw up{block_idx} [{ch}->{ch}, k=1, T={frames}]");
- let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_in");
- let weight = context.create_array_zeros(&[ch * ch], dt, "pw_w");
- let bias = context.create_array_zeros(&[ch], dt, "pw_b");
- let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_out");
+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt);
+ let weight = context.create_array_zeros(&[ch * ch], dt);
+ let bias = context.create_array_zeros(&[ch], dt);
+ let mut output = context.create_array_zeros(&[batch_size * ch * frames], dt).into_allocation();
let lengths = make_lengths(&context, batch_size, frames as i32);
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let w = weight.buffer();
- let w = w.borrow();
- let b = bias.buffer();
- let b = b.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- let l = lengths.buffer();
- let l = l.borrow();
+ let i = input.allocation();
+ let w = weight.allocation();
+ let b = bias.allocation();
+ let o = &mut output;
+ let l = lengths.allocation();
k_conv1d.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
ch as i32,
ch as i32,
frames as i32,
@@ -765,17 +699,15 @@ fn audio_kernel_perf() {
{
let n = ch * frames;
let label = format!("GELU up{block_idx} [n={n}]");
- let input = context.create_array_zeros(&[n], dt, "gelu_in");
- let output = context.create_array_zeros(&[n], dt, "gelu_out");
+ let input = context.create_array_zeros(&[n], dt);
+ let mut output = context.create_array_zeros(&[n], dt).into_allocation();
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
{
- let i = input.buffer();
- let i = i.borrow();
- let o = output.buffer();
- let mut o = o.borrow_mut();
- k_act.encode(Some(&*i), &mut *o, n as u32, ActivationType::GELU, &mut encoder);
+ let i = input.allocation();
+ let o = &mut output;
+ k_act.encode(Some(i), o, n as u32, ActivationType::GELU, &mut encoder);
}
encoder.end_encoding().submit().wait_until_completed().unwrap();
});
@@ -813,15 +745,14 @@ fn audio_kernel_perf() {
let frames = SEMANTIC_FRAMES_10S;
// Quantizer arrays
- let p_q_tok = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");
+ let p_q_tok = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32);
let p_q_len = make_lengths(&context, batch_size, frames as i32);
- let p_q_sem_cb = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");
- let p_q_sem_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");
- let p_q_sem_bias = context.create_array_zeros(&[INPUT_DIM], dt, "p_sem_bias");
- let p_q_res_cbs =
- context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");
- let p_q_res_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");
- let p_q_res_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");
+ let p_q_sem_cb = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt);
+ let p_q_sem_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt);
+ let p_q_sem_bias = context.create_array_zeros(&[INPUT_DIM], dt);
+ let p_q_res_cbs = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt);
+ let p_q_res_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt);
+ let p_q_res_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt);
// Compute frame counts at each stage
let f_up0 = frames * UP_STRIDES[0]; // 430
@@ -834,42 +765,41 @@ fn audio_kernel_perf() {
// Pipeline scratch: large enough for the biggest intermediate
// Biggest is decoder_dim * f_up1 or ch * f_dec stages
let max_elems: usize = batch_size * DECODER_DIM * f_dec3.max(f_up1);
- let scratch_a = context.create_array_zeros(&[max_elems], dt, "pipe_sa");
- let scratch_b = context.create_array_zeros(&[max_elems], dt, "pipe_sb");
- // Third scratch buffer for residual connections (avoids RefCell borrow conflict)
- let scratch_r = context.create_array_zeros(&[max_elems], dt, "pipe_sr");
+ let mut scratch_a = context.create_array_zeros(&[max_elems], dt).into_allocation();
+ let mut scratch_b = context.create_array_zeros(&[max_elems], dt).into_allocation();
+ let scratch_r = context.create_array_zeros(&[max_elems], dt).into_allocation();
// Upsample block 0: 1024->512, stride=2
- let u0_w = context.create_array_zeros(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");
- let u0_b = context.create_array_zeros(&[UP_CHANNELS[1]], dt, "u0_b");
+ let u0_w = context.create_array_zeros(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt);
+ let u0_b = context.create_array_zeros(&[UP_CHANNELS[1]], dt);
// ConvNeXt 0 (ch=512)
let c0_ch = UP_CHANNELS[1];
- let c0_dw_w = context.create_array_zeros(&[c0_ch * 1 * 7], dt, "c0_dw_w");
- let c0_dw_b = context.create_array_zeros(&[c0_ch], dt, "c0_dw_b");
- let c0_ns = context.create_array_zeros(&[c0_ch], dt, "c0_ns");
- let c0_nb = context.create_array_zeros(&[c0_ch], dt, "c0_nb");
- let c0_p1_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p1_w");
- let c0_p1_b = context.create_array_zeros(&[c0_ch], dt, "c0_p1_b");
- let c0_p2_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p2_w");
- let c0_p2_b = context.create_array_zeros(&[c0_ch], dt, "c0_p2_b");
+ let c0_dw_w = context.create_array_zeros(&[c0_ch * 1 * 7], dt);
+ let c0_dw_b = context.create_array_zeros(&[c0_ch], dt);
+ let c0_ns = context.create_array_zeros(&[c0_ch], dt);
+ let c0_nb = context.create_array_zeros(&[c0_ch], dt);
+ let c0_p1_w = context.create_array_zeros(&[c0_ch * c0_ch], dt);
+ let c0_p1_b = context.create_array_zeros(&[c0_ch], dt);
+ let c0_p2_w = context.create_array_zeros(&[c0_ch * c0_ch], dt);
+ let c0_p2_b = context.create_array_zeros(&[c0_ch], dt);
// Upsample block 1: 512->256, stride=2
- let u1_w = context.create_array_zeros(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");
- let u1_b = context.create_array_zeros(&[UP_CHANNELS[2]], dt, "u1_b");
+ let u1_w = context.create_array_zeros(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt);
+ let u1_b = context.create_array_zeros(&[UP_CHANNELS[2]], dt);
// ConvNeXt 1 (ch=256)
let c1_ch = UP_CHANNELS[2];
- let c1_dw_w = context.create_array_zeros(&[c1_ch * 1 * 7], dt, "c1_dw_w");
- let c1_dw_b = context.create_array_zeros(&[c1_ch], dt, "c1_dw_b");
- let c1_ns = context.create_array_zeros(&[c1_ch], dt, "c1_ns");
- let c1_nb = context.create_array_zeros(&[c1_ch], dt, "c1_nb");
- let c1_p1_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p1_w");
- let c1_p1_b = context.create_array_zeros(&[c1_ch], dt, "c1_p1_b");
- let c1_p2_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p2_w");
- let c1_p2_b = context.create_array_zeros(&[c1_ch], dt, "c1_p2_b");
+ let c1_dw_w = context.create_array_zeros(&[c1_ch * 1 * 7], dt);
+ let c1_dw_b = context.create_array_zeros(&[c1_ch], dt);
+ let c1_ns = context.create_array_zeros(&[c1_ch], dt);
+ let c1_nb = context.create_array_zeros(&[c1_ch], dt);
+ let c1_p1_w = context.create_array_zeros(&[c1_ch * c1_ch], dt);
+ let c1_p1_b = context.create_array_zeros(&[c1_ch], dt);
+ let c1_p2_w = context.create_array_zeros(&[c1_ch * c1_ch], dt);
+ let c1_p2_b = context.create_array_zeros(&[c1_ch], dt);
// first_conv: 256->1536, k=7
- let fc_w = context.create_array_zeros(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");
- let fc_b = context.create_array_zeros(&[DECODER_DIM], dt, "fc_b");
+ let fc_w = context.create_array_zeros(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt);
+ let fc_b = context.create_array_zeros(&[DECODER_DIM], dt);
// Decoder block weights for all 4 blocks
struct DecBlockWeights {
@@ -892,27 +822,23 @@ fn audio_kernel_perf() {
let stride = DEC_STRIDES[i];
let ksize = stride * 2;
DecBlockWeights {
- snake_a: context.create_array_zeros(&[cin], dt, &format!("d{i}_sa")),
- tconv_w: context.create_array_zeros(&[cin * cout * ksize], dt, &format!("d{i}_tw")),
- tconv_b: context.create_array_zeros(&[cout], dt, &format!("d{i}_tb")),
- ru_s1: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s1"))),
- ru_c1_w: std::array::from_fn(|j| {
- context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))
- }),
- ru_c1_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c1b"))),
- ru_s2: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s2"))),
- ru_c2_w: std::array::from_fn(|j| {
- context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))
- }),
- ru_c2_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c2b"))),
+ snake_a: context.create_array_zeros(&[cin], dt),
+ tconv_w: context.create_array_zeros(&[cin * cout * ksize], dt),
+ tconv_b: context.create_array_zeros(&[cout], dt),
+ ru_s1: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),
+ ru_c1_w: std::array::from_fn(|_| context.create_array_zeros(&[cout * cout * 7], dt)),
+ ru_c1_b: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),
+ ru_s2: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),
+ ru_c2_w: std::array::from_fn(|_| context.create_array_zeros(&[cout * cout * 7], dt)),
+ ru_c2_b: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),
}
})
.collect();
// final snake + conv + tanh
- let final_snake_a = context.create_array_zeros(&[DEC_CHANNELS[4]], dt, "final_sa");
- let final_conv_w = context.create_array_zeros(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");
- let final_conv_b = context.create_array_zeros(&[1], dt, "final_cb");
+ let final_snake_a = context.create_array_zeros(&[DEC_CHANNELS[4]], dt);
+ let final_conv_w = context.create_array_zeros(&[1 * DEC_CHANNELS[4] * 7], dt);
+ let final_conv_b = context.create_array_zeros(&[1], dt);
// Length arrays
let len_up0 = make_lengths(&context, batch_size, f_up0 as i32);
@@ -927,38 +853,27 @@ fn audio_kernel_perf() {
let ms = measure("Full pipeline (quant -> 2 up -> first_conv -> 4 dec -> final)", || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
- let sa = scratch_a.buffer();
- let sb = scratch_b.buffer();
// --- Quantizer: output [B, T, input_dim] (NSC) ---
{
- let tok = p_q_tok.buffer();
- let tok = tok.borrow();
- let len = p_q_len.buffer();
- let len = len.borrow();
- let scb = p_q_sem_cb.buffer();
- let scb = scb.borrow();
- let sp = p_q_sem_proj.buffer();
- let sp = sp.borrow();
- let sbi = p_q_sem_bias.buffer();
- let sbi = sbi.borrow();
- let rcb = p_q_res_cbs.buffer();
- let rcb = rcb.borrow();
- let rp = p_q_res_proj.buffer();
- let rp = rp.borrow();
- let rb = p_q_res_bias.buffer();
- let rb = rb.borrow();
- let mut out = sa.borrow_mut();
+ let tok = p_q_tok.allocation();
+ let len = p_q_len.allocation();
+ let scb = p_q_sem_cb.allocation();
+ let sp = p_q_sem_proj.allocation();
+ let sbi = p_q_sem_bias.allocation();
+ let rcb = p_q_res_cbs.allocation();
+ let rp = p_q_res_proj.allocation();
+ let rb = p_q_res_bias.allocation();
k_quant.encode(
- &*tok,
- &*len,
- &*scb,
- &*sp,
- &*sbi,
- &*rcb,
- &*rp,
- &*rb,
- &mut *out,
+ tok,
+ len,
+ scb,
+ sp,
+ sbi,
+ rcb,
+ rp,
+ rb,
+ &mut scratch_a,
batch_size as i32,
TOTAL_CODEBOOKS as i32,
frames as i32,
@@ -973,20 +888,15 @@ fn audio_kernel_perf() {
// --- Upsample block 0: 1024->512, stride=2, NSC input ---
{
- let i = sa.borrow();
- let w = u0_w.buffer();
- let w = w.borrow();
- let b = u0_b.buffer();
- let b = b.borrow();
- let mut o = sb.borrow_mut();
- let l = len_up0.buffer();
- let l = l.borrow();
+ let w = u0_w.allocation();
+ let b = u0_b.allocation();
+ let l = len_up0.allocation();
k_tconv.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_a,
+ w,
+ b,
+ &mut scratch_b,
+ l,
UP_CHANNELS[0] as i32,
UP_CHANNELS[1] as i32,
frames as i32,
@@ -1001,20 +911,15 @@ fn audio_kernel_perf() {
}
// ConvNeXt 0
{
- let i = sb.borrow();
- let w = c0_dw_w.buffer();
- let w = w.borrow();
- let b = c0_dw_b.buffer();
- let b = b.borrow();
- let mut o = sa.borrow_mut();
- let l = len_up0.buffer();
- let l = l.borrow();
+ let w = c0_dw_w.allocation();
+ let b = c0_dw_b.allocation();
+ let l = len_up0.allocation();
k_conv_grp.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_b,
+ w,
+ b,
+ &mut scratch_a,
+ l,
c0_ch as i32,
c0_ch as i32,
f_up0 as i32,
@@ -1027,20 +932,15 @@ fn audio_kernel_perf() {
);
}
{
- let i = sa.borrow();
- let s = c0_ns.buffer();
- let s = s.borrow();
- let b = c0_nb.buffer();
- let b = b.borrow();
- let mut o = sb.borrow_mut();
- let l = len_up0.buffer();
- let l = l.borrow();
+ let s = c0_ns.allocation();
+ let b = c0_nb.allocation();
+ let l = len_up0.allocation();
k_norm.encode(
- &*i,
- &*s,
- &*b,
- &mut *o,
- &*l,
+ &scratch_a,
+ s,
+ b,
+ &mut scratch_b,
+ l,
c0_ch as i32,
f_up0 as i32,
1e-6_f32,
@@ -1050,20 +950,15 @@ fn audio_kernel_perf() {
);
}
{
- let i = sb.borrow();
- let w = c0_p1_w.buffer();
- let w = w.borrow();
- let b = c0_p1_b.buffer();
- let b = b.borrow();
- let mut o = sa.borrow_mut();
- let l = len_up0.buffer();
- let l = l.borrow();
+ let w = c0_p1_w.allocation();
+ let b = c0_p1_b.allocation();
+ let l = len_up0.allocation();
k_conv1d.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_b,
+ w,
+ b,
+ &mut scratch_a,
+ l,
c0_ch as i32,
c0_ch as i32,
f_up0 as i32,
@@ -1078,25 +973,18 @@ fn audio_kernel_perf() {
);
}
{
- let i = sa.borrow();
- let mut o = sb.borrow_mut();
- k_act.encode(Some(&*i), &mut *o, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);
+ k_act.encode(Some(&scratch_a), &mut scratch_b, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);
}
{
- let i = sb.borrow();
- let w = c0_p2_w.buffer();
- let w = w.borrow();
- let b = c0_p2_b.buffer();
- let b = b.borrow();
- let mut o = sa.borrow_mut();
- let l = len_up0.buffer();
- let l = l.borrow();
+ let w = c0_p2_w.allocation();
+ let b = c0_p2_b.allocation();
+ let l = len_up0.allocation();
k_conv1d.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_b,
+ w,
+ b,
+ &mut scratch_a,
+ l,
c0_ch as i32,
c0_ch as i32,
f_up0 as i32,
@@ -1111,27 +999,20 @@ fn audio_kernel_perf() {
);
}
{
- let a = sa.borrow();
- let mut o = sb.borrow_mut();
- k_add.encode(&*a, &*a, &mut *o, (c0_ch * f_up0) as i32, &mut encoder);
+ k_add.encode(&scratch_a, &scratch_a, &mut scratch_b, (c0_ch * f_up0) as i32, &mut encoder);
}
// --- Upsample block 1: 512->256, stride=2, NCS input ---
{
- let i = sb.borrow();
- let w = u1_w.buffer();
- let w = w.borrow();
- let b = u1_b.buffer();
- let b = b.borrow();
- let mut o = sa.borrow_mut();
- let l = len_up1.buffer();
- let l = l.borrow();
+ let w = u1_w.allocation();
+ let b = u1_b.allocation();
+ let l = len_up1.allocation();
k_tconv.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_b,
+ w,
+ b,
+ &mut scratch_a,
+ l,
UP_CHANNELS[1] as i32,
UP_CHANNELS[2] as i32,
f_up0 as i32,
@@ -1146,20 +1027,15 @@ fn audio_kernel_perf() {
}
// ConvNeXt 1
{
- let i = sa.borrow();
- let w = c1_dw_w.buffer();
- let w = w.borrow();
- let b = c1_dw_b.buffer();
- let b = b.borrow();
- let mut o = sb.borrow_mut();
- let l = len_up1.buffer();
- let l = l.borrow();
+ let w = c1_dw_w.allocation();
+ let b = c1_dw_b.allocation();
+ let l = len_up1.allocation();
k_conv_grp.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_a,
+ w,
+ b,
+ &mut scratch_b,
+ l,
c1_ch as i32,
c1_ch as i32,
f_up1 as i32,
@@ -1172,20 +1048,15 @@ fn audio_kernel_perf() {
);
}
{
- let i = sb.borrow();
- let s = c1_ns.buffer();
- let s = s.borrow();
- let b = c1_nb.buffer();
- let b = b.borrow();
- let mut o = sa.borrow_mut();
- let l = len_up1.buffer();
- let l = l.borrow();
+ let s = c1_ns.allocation();
+ let b = c1_nb.allocation();
+ let l = len_up1.allocation();
k_norm.encode(
- &*i,
- &*s,
- &*b,
- &mut *o,
- &*l,
+ &scratch_b,
+ s,
+ b,
+ &mut scratch_a,
+ l,
c1_ch as i32,
f_up1 as i32,
1e-6_f32,
@@ -1195,20 +1066,15 @@ fn audio_kernel_perf() {
);
}
{
- let i = sa.borrow();
- let w = c1_p1_w.buffer();
- let w = w.borrow();
- let b = c1_p1_b.buffer();
- let b = b.borrow();
- let mut o = sb.borrow_mut();
- let l = len_up1.buffer();
- let l = l.borrow();
+ let w = c1_p1_w.allocation();
+ let b = c1_p1_b.allocation();
+ let l = len_up1.allocation();
k_conv1d.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_a,
+ w,
+ b,
+ &mut scratch_b,
+ l,
c1_ch as i32,
c1_ch as i32,
f_up1 as i32,
@@ -1223,25 +1089,18 @@ fn audio_kernel_perf() {
);
}
{
- let i = sb.borrow();
- let mut o = sa.borrow_mut();
- k_act.encode(Some(&*i), &mut *o, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);
+ k_act.encode(Some(&scratch_b), &mut scratch_a, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);
}
{
- let i = sa.borrow();
- let w = c1_p2_w.buffer();
- let w = w.borrow();
- let b = c1_p2_b.buffer();
- let b = b.borrow();
- let mut o = sb.borrow_mut();
- let l = len_up1.buffer();
- let l = l.borrow();
+ let w = c1_p2_w.allocation();
+ let b = c1_p2_b.allocation();
+ let l = len_up1.allocation();
k_conv1d.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_a,
+ w,
+ b,
+ &mut scratch_b,
+ l,
c1_ch as i32,
c1_ch as i32,
f_up1 as i32,
@@ -1256,27 +1115,20 @@ fn audio_kernel_perf() {
);
}
{
- let a = sb.borrow();
- let mut o = sa.borrow_mut();
- k_add.encode(&*a, &*a, &mut *o, (c1_ch * f_up1) as i32, &mut encoder);
+ k_add.encode(&scratch_b, &scratch_b, &mut scratch_a, (c1_ch * f_up1) as i32, &mut encoder);
}
// --- first_conv: 256->1536, k=7 ---
{
- let i = sa.borrow();
- let w = fc_w.buffer();
- let w = w.borrow();
- let b = fc_b.buffer();
- let b = b.borrow();
- let mut o = sb.borrow_mut();
- let l = len_up1.buffer();
- let l = l.borrow();
+ let w = fc_w.allocation();
+ let b = fc_b.allocation();
+ let l = len_up1.allocation();
k_conv1d_causal.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ &scratch_a,
+ w,
+ b,
+ &mut scratch_b,
+ l,
UP_CHANNELS[2] as i32,
DECODER_DIM as i32,
f_up1 as i32,
@@ -1304,19 +1156,16 @@ fn audio_kernel_perf() {
// snake (in-place swap)
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let a = blk.snake_a.buffer();
- let a = a.borrow();
- let mut o = o_buf.borrow_mut();
+ let a = blk.snake_a.allocation();
k_snake.encode(
- &*i,
- &*a,
- &mut *o,
+ i,
+ a,
+ o,
cin as i32,
cur_frames as i32,
cin as i32,
@@ -1330,25 +1179,20 @@ fn audio_kernel_perf() {
// trans_conv
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let w = blk.tconv_w.buffer();
- let w = w.borrow();
- let b = blk.tconv_b.buffer();
- let b = b.borrow();
- let mut o = o_buf.borrow_mut();
- let l = dec_len.buffer();
- let l = l.borrow();
+ let w = blk.tconv_w.allocation();
+ let b = blk.tconv_b.allocation();
+ let l = dec_len.allocation();
k_tconv.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
cin as i32,
cout as i32,
cur_frames as i32,
@@ -1365,23 +1209,19 @@ fn audio_kernel_perf() {
cur_frames = next_frames;
// 3 residual units
- let sr = scratch_r.buffer();
for ru_idx in 0..3 {
// snake1
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let a = blk.ru_s1[ru_idx].buffer();
- let a = a.borrow();
- let mut o = o_buf.borrow_mut();
+ let a = blk.ru_s1[ru_idx].allocation();
k_snake.encode(
- &*i,
- &*a,
- &mut *o,
+ i,
+ a,
+ o,
cout as i32,
cur_frames as i32,
cout as i32,
@@ -1395,25 +1235,20 @@ fn audio_kernel_perf() {
// conv1 (dilation varies: 1, 3, 9)
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let w = blk.ru_c1_w[ru_idx].buffer();
- let w = w.borrow();
- let b = blk.ru_c1_b[ru_idx].buffer();
- let b = b.borrow();
- let mut o = o_buf.borrow_mut();
- let l = dec_len.buffer();
- let l = l.borrow();
+ let w = blk.ru_c1_w[ru_idx].allocation();
+ let b = blk.ru_c1_b[ru_idx].allocation();
+ let l = dec_len.allocation();
k_conv1d_causal.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
cout as i32,
cout as i32,
cur_frames as i32,
@@ -1428,19 +1263,16 @@ fn audio_kernel_perf() {
// snake2
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let a = blk.ru_s2[ru_idx].buffer();
- let a = a.borrow();
- let mut o = o_buf.borrow_mut();
+ let a = blk.ru_s2[ru_idx].allocation();
k_snake.encode(
- &*i,
- &*a,
- &mut *o,
+ i,
+ a,
+ o,
cout as i32,
cur_frames as i32,
cout as i32,
@@ -1453,30 +1285,23 @@ fn audio_kernel_perf() {
cur_in_sa = !cur_in_sa;
// conv2 + residual add
- // Use scratch_r as the residual source (separate buffer to
- // avoid RefCell borrow conflict with the output destination).
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let r = sr.borrow();
- let w = blk.ru_c2_w[ru_idx].buffer();
- let w = w.borrow();
- let b = blk.ru_c2_b[ru_idx].buffer();
- let b = b.borrow();
- let mut o = o_buf.borrow_mut();
- let l = dec_len.buffer();
- let l = l.borrow();
+ let r = &scratch_r;
+ let w = blk.ru_c2_w[ru_idx].allocation();
+ let b = blk.ru_c2_b[ru_idx].allocation();
+ let l = dec_len.allocation();
k_conv_res.encode(
- &*i,
- &*r,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ r,
+ w,
+ b,
+ o,
+ l,
cout as i32,
cout as i32,
cur_frames as i32,
@@ -1493,19 +1318,16 @@ fn audio_kernel_perf() {
// --- final snake + conv + tanh ---
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let a = final_snake_a.buffer();
- let a = a.borrow();
- let mut o = o_buf.borrow_mut();
+ let a = final_snake_a.allocation();
k_snake.encode(
- &*i,
- &*a,
- &mut *o,
+ i,
+ a,
+ o,
DEC_CHANNELS[4] as i32,
f_dec3 as i32,
DEC_CHANNELS[4] as i32,
@@ -1517,25 +1339,20 @@ fn audio_kernel_perf() {
}
cur_in_sa = !cur_in_sa;
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let w = final_conv_w.buffer();
- let w = w.borrow();
- let b = final_conv_b.buffer();
- let b = b.borrow();
- let mut o = o_buf.borrow_mut();
- let l = len_dec3.buffer();
- let l = l.borrow();
+ let w = final_conv_w.allocation();
+ let b = final_conv_b.allocation();
+ let l = len_dec3.allocation();
k_conv1d_causal.encode(
- &*i,
- &*w,
- &*b,
- &mut *o,
- &*l,
+ i,
+ w,
+ b,
+ o,
+ l,
DEC_CHANNELS[4] as i32,
1_i32,
f_dec3 as i32,
@@ -1549,14 +1366,12 @@ fn audio_kernel_perf() {
cur_in_sa = !cur_in_sa;
// tanh
{
- let (i_buf, o_buf) = if cur_in_sa {
- (sa.clone(), sb.clone())
+ let (i, o) = if cur_in_sa {
+ (&scratch_a, &mut scratch_b)
} else {
- (sb.clone(), sa.clone())
+ (&scratch_b, &mut scratch_a)
};
- let i = i_buf.borrow();
- let mut o = o_buf.borrow_mut();
- k_act.encode(Some(&*i), &mut *o, f_dec3 as u32, ActivationType::TANH, &mut encoder);
+ k_act.encode(Some(i), o, f_dec3 as u32, ActivationType::TANH, &mut encoder);
}
encoder.end_encoding().submit().wait_until_completed().unwrap();
diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs
index 052b6e371..47a50d96e 100644
--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs
+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs
@@ -13,7 +13,7 @@ use backend_uzu::{
use half::bf16;
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use crate::common::helpers::{alloc_buffer, alloc_buffer_with_data, create_context};
+use crate::common::helpers::{alloc_allocation, alloc_allocation_with_data, create_context};
fn build_offsets(
e: usize,
@@ -77,99 +77,98 @@ fn run_decode_case<B: Backend>(
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(ctx).expect("experts decode kernel");
- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x_perm);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);
+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
let num_tiles_k = d_ff.div_ceil(K_TILE);
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);
- let mut output_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);
-
// Buffers for indirect dispatch
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
let max_tiles = e * sum_k * 16384; // Conservative upper bound
- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);
- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, sum_k);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- experts_kernel.encode(
- &mut encoder,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut output_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- data_type: DataType::BF16,
- },
- );
+ let output = experts_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsTwoPassArguments {
+ x_perm: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &mut row_expert_map_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &mut tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ tile_map: &mut tile_map_buf,
+ total_tiles: &mut total_tiles_buf,
+ dispatch_args: &mut dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.702,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- experts_kernel.encode(
- &mut encoder,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut output_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- data_type: DataType::BF16,
- },
- );
+ let output = experts_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsTwoPassArguments {
+ x_perm: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &mut row_expert_map_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &mut tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ tile_map: &mut tile_map_buf,
+ total_tiles: &mut total_tiles_buf,
+ dispatch_args: &mut dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.702,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -227,34 +226,29 @@ fn run_two_pass_prefill_case<B: Backend>(
let experts_kernel = MoeExpertsTwoPassPrefillBlock::<B>::new(ctx).expect("experts prefill kernel");
- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x_perm);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);
+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
let num_tiles_k = d_ff.div_ceil(K_TILE);
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);
- let mut output_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);
-
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
let max_tiles = e * sum_k * 16384;
- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);
- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, sum_k);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
+ x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut output_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
@@ -277,8 +271,9 @@ fn run_two_pass_prefill_case<B: Backend>(
silu_alpha: 1.702,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut encoder, args);
+ let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
}
let mut times = Vec::with_capacity(iters);
@@ -286,11 +281,9 @@ fn run_two_pass_prefill_case<B: Backend>(
let start = Instant::now();
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
+ x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut output_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
@@ -313,8 +306,9 @@ fn run_two_pass_prefill_case<B: Backend>(
silu_alpha: 1.702,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut encoder, args);
+ let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -362,74 +356,73 @@ fn run_fused_single_token_case<B: Backend>(
let fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(ctx).expect("fused kernel");
- let x_buf = alloc_buffer_with_data::<B, bf16>(ctx, &x);
- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(ctx, &topk_ids);
- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(ctx, &topk_probs);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w13_all);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w2_all);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &down_biases);
- let mut hidden_buf = alloc_buffer::<B, f32>(ctx, k * d_ff);
- let mut y_buf = alloc_buffer::<B, bf16>(ctx, d_model);
-
+ let x_buf = alloc_allocation_with_data::<B, bf16>(ctx, &x);
+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(ctx, &topk_ids);
+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(ctx, &topk_probs);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w13_all);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w2_all);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- fused_kernel.encode(
- &mut encoder,
- MoeExpertsSingleDecodeArguments {
- x: &x_buf,
- topk_ids: &topk_ids_buf,
- topk_probs: &topk_probs_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- hidden: &mut hidden_buf,
- y: &mut y_buf,
- d_model,
- d_ff,
- k,
- gating_code: 2, // SwiGLU
- silu_alpha: 1.0,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- data_type: DataType::BF16,
- },
- );
+ let output = fused_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code: 2, // SwiGLU
+ silu_alpha: 1.0,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- fused_kernel.encode(
- &mut encoder,
- MoeExpertsSingleDecodeArguments {
- x: &x_buf,
- topk_ids: &topk_ids_buf,
- topk_probs: &topk_probs_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- hidden: &mut hidden_buf,
- y: &mut y_buf,
- d_model,
- d_ff,
- k,
- gating_code: 2, // SwiGLU
- silu_alpha: 1.0,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- data_type: DataType::BF16,
- },
- );
+ let output = fused_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code: 2, // SwiGLU
+ silu_alpha: 1.0,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -484,97 +477,96 @@ fn run_indirect_decode_timed<B: Backend>(
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(ctx).expect("decode kernel");
- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(ctx, &x_perm);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(ctx, &offsets);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w13);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w2);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &down_biases);
+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(ctx, &x_perm);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(ctx, &offsets);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w13);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w2);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);
let num_tiles_k = d_ff.div_ceil(K_TILE);
- let mut hidden_buf = alloc_buffer::<B, f32>(ctx, sum_k * d_ff);
- let mut output_buf = alloc_buffer::<B, bf16>(ctx, sum_k * d_model);
-
- let mut tile_counts_buf = alloc_buffer::<B, u32>(ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(ctx, e + 1);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(ctx, e + 1);
let max_tiles = e * sum_k * 16384;
- let mut tile_map_buf = alloc_buffer::<B, u32>(ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(ctx, 1);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(ctx, 3);
- let mut row_expert_map_buf = alloc_buffer::<B, u32>(ctx, sum_k);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(ctx, max_tiles * 3);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(ctx, 1);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(ctx, 3);
+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(ctx, sum_k);
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- experts_kernel.encode(
- &mut encoder,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut output_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.0,
- data_type: DataType::BF16,
- },
- );
+ let output = experts_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsTwoPassArguments {
+ x_perm: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &mut row_expert_map_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &mut tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ tile_map: &mut tile_map_buf,
+ total_tiles: &mut total_tiles_buf,
+ dispatch_args: &mut dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.0,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- experts_kernel.encode(
- &mut encoder,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut output_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.0,
- data_type: DataType::BF16,
- },
- );
+ let output = experts_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsTwoPassArguments {
+ x_perm: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &mut row_expert_map_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &mut tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ tile_map: &mut tile_map_buf,
+ total_tiles: &mut total_tiles_buf,
+ dispatch_args: &mut dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.0,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -610,74 +602,73 @@ fn run_fused_decode_timed<B: Backend>(
let fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(ctx).expect("fused kernel");
- let x_buf = alloc_buffer_with_data::<B, bf16>(ctx, &x);
- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(ctx, &topk_ids);
- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(ctx, &topk_probs);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w13_all);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w2_all);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &down_biases);
- let mut hidden_buf = alloc_buffer::<B, f32>(ctx, k * d_ff);
- let mut y_buf = alloc_buffer::<B, bf16>(ctx, d_model);
-
+ let x_buf = alloc_allocation_with_data::<B, bf16>(ctx, &x);
+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(ctx, &topk_ids);
+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(ctx, &topk_probs);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w13_all);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w2_all);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);
for _ in 0..warmup {
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- fused_kernel.encode(
- &mut encoder,
- MoeExpertsSingleDecodeArguments {
- x: &x_buf,
- topk_ids: &topk_ids_buf,
- topk_probs: &topk_probs_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- hidden: &mut hidden_buf,
- y: &mut y_buf,
- d_model,
- d_ff,
- k,
- gating_code: 2,
- silu_alpha: 1.0,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- data_type: DataType::BF16,
- },
- );
+ let output = fused_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code: 2,
+ silu_alpha: 1.0,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
- fused_kernel.encode(
- &mut encoder,
- MoeExpertsSingleDecodeArguments {
- x: &x_buf,
- topk_ids: &topk_ids_buf,
- topk_probs: &topk_probs_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- hidden: &mut hidden_buf,
- y: &mut y_buf,
- d_model,
- d_ff,
- k,
- gating_code: 2,
- silu_alpha: 1.0,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- data_type: DataType::BF16,
- },
- );
+ let output = fused_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code: 2,
+ silu_alpha: 1.0,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs
index af7db6b20..355dec81a 100644
--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs
+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs
@@ -13,7 +13,7 @@ use half::bf16;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::common::{
- helpers::{alloc_buffer, alloc_buffer_with_data, create_context},
+ helpers::{alloc_allocation, alloc_allocation_with_data, create_context},
perf::run_perf_with_warmup,
};
@@ -39,11 +39,11 @@ fn test_moe_e2e_decode_perf() {
let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
// Buffers (simplified - just key buffers for timing)
- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);
- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_w);
- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_b);
- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);
- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);
+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);
+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_w);
+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_b);
+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);
+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);
let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
.expect("router+topk fused kernel");
@@ -96,11 +96,11 @@ fn test_moe_e2e_prefill_perf() {
let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
// Buffers (simplified - just key buffers for timing)
- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);
- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_w);
- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_b);
- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);
- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);
+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);
+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_w);
+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_b);
+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);
+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);
let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
.expect("router+topk fused kernel");
@@ -151,21 +151,22 @@ fn test_moe_pipeline_breakdown_decode() {
let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();
let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);
- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_w);
- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_b);
- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);
- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);
+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);
+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_w);
+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_b);
+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);
+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);
let num_blocks = ((t + 255) / 256).max(1);
let num_tiles = ((e + 512 - 1) / 512).max(1);
- let mut partials_buf = alloc_buffer::<B, i32>(&ctx, num_blocks * num_tiles * e);
- let mut offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut sumk_buf = alloc_buffer::<B, u32>(&ctx, 1);
- let mut bucketed_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);
- let mut x_perm_buf = alloc_buffer::<B, bf16>(&ctx, t * k * d_model);
- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, t * k * d_model);
- let mut tok2row_buf = alloc_buffer::<B, i32>(&ctx, t * k);
- let mut y_out_buf = alloc_buffer::<B, bf16>(&ctx, t * d_model);
+ let mut partials_buf = alloc_allocation::<B, i32>(&ctx, num_blocks * num_tiles * e);
+ let mut offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut sumk_buf = alloc_allocation::<B, u32>(&ctx, 1);
+ let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);
+ let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, t * k * d_model);
+ let y_partial: Vec<bf16> = (0..t * k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();
+ let y_partial_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &y_partial);
+ let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);
+ let mut y_out_buf = alloc_allocation::<B, bf16>(&ctx, t * d_model);
// Expert weights buffers
// Generate W13 in original layout [E, d_model, 2*d_ff]
@@ -190,30 +191,28 @@ fn test_moe_pipeline_breakdown_decode() {
let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
// Experts tiling buffers for two-pass
const K_TILE: usize = 64;
let sum_k = t * k;
let num_tiles_k = (d_ff + K_TILE - 1) / K_TILE;
let max_tiles = t * k * e * num_tiles_k;
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
- // Two-pass specific buffers
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);
- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, sum_k);
+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);
// Scatter block bases buffers
- let mut block_bases_buf = alloc_buffer::<B, u32>(&ctx, num_blocks * num_tiles);
- let mut block_alloc_buf = alloc_buffer::<B, u32>(&ctx, num_blocks * num_tiles);
- let mut bucketed_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);
+ let mut block_bases_buf = alloc_allocation::<B, u32>(&ctx, num_blocks * num_tiles);
+ let mut block_alloc_buf = alloc_allocation::<B, u32>(&ctx, num_blocks * num_tiles);
+ let mut bucketed_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);
// Create kernel structs (use production-validated encoding logic)
let counts_offsets_kernel =
@@ -302,10 +301,10 @@ fn test_moe_pipeline_breakdown_decode() {
&mut encoder,
DataType::BF16,
MoeGatherArguments {
- x_buffer: &x_buf,
- bucketed_ids_buffer: &bucketed_ids_buf,
- x_perm_buffer: &mut x_perm_buf,
- sumk_buffer: &sumk_buf,
+ x: &x_buf,
+ bucketed_ids: &bucketed_ids_buf,
+ x_perm: &mut x_perm_buf,
+ sumk: &sumk_buf,
t,
k,
d_model,
@@ -316,38 +315,39 @@ fn test_moe_pipeline_breakdown_decode() {
let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- experts_kernel.encode(
- &mut encoder,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut y_partial_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2, // SILU
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: 20.0,
- up_clip_min: -19.0,
- up_clip_max: 21.0,
- silu_alpha: 1.702,
- data_type: DataType::BF16,
- },
- );
+ let output = experts_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsTwoPassArguments {
+ x_perm: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &mut row_expert_map_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &mut tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ tile_map: &mut tile_map_buf,
+ total_tiles: &mut total_tiles_buf,
+ dispatch_args: &mut dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2, // SILU
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: 20.0,
+ up_clip_min: -19.0,
+ up_clip_max: 21.0,
+ silu_alpha: 1.702,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
encoder.end_encoding().submit().wait_until_completed().unwrap();
+ drop(output);
});
let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {
diff --git a/crates/backend-uzu/tests/performance/matmul/bench.rs b/crates/backend-uzu/tests/performance/matmul/bench.rs
index c350907dc..d21ef12ae 100644
--- a/crates/backend-uzu/tests/performance/matmul/bench.rs
+++ b/crates/backend-uzu/tests/performance/matmul/bench.rs
@@ -4,7 +4,7 @@ use backend_uzu::{
DataType,
backends::{
common::{
- Backend, Encoder,
+ Allocation, AllocationType, AsBufferRangeMut, Backend, Context, DenseBuffer, Encoder,
kernel::{
ManualKernels,
matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
@@ -13,23 +13,26 @@ use backend_uzu::{
metal::Metal,
},
};
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{error::BenchError, output::PerfResult, shapes::TestShape};
type Ctx = <Metal as Backend>::Context;
-type Buf = Retained<ProtocolObject<dyn MTLBuffer>>;
const WARMUP_ITERATIONS: usize = 3;
const BENCHMARK_ITERATIONS: usize = 10;
fn fill_buffer_random(
- buffer: &ProtocolObject<dyn MTLBuffer>,
+ allocation: &mut Allocation<Metal>,
byte_count: usize,
) {
- let pointer = buffer.contents().as_ptr() as *mut u8;
- let slice = unsafe { std::slice::from_raw_parts_mut(pointer, byte_count) };
+ let buffer_range = allocation.as_buffer_range_mut();
+ let range = buffer_range.range();
+ let slice = unsafe {
+ std::slice::from_raw_parts_mut(
+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),
+ byte_count,
+ )
+ };
for (i, byte) in slice.iter_mut().enumerate() {
*byte = (i % 251) as u8;
}
@@ -39,21 +42,20 @@ fn encode_and_run(
context: &Ctx,
kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
shape: &TestShape,
- a_buffer: &Buf,
- b_buffer: &Buf,
- d_buffer: &mut Buf,
+ a_allocation: &backend_uzu::backends::common::Allocation<Metal>,
+ b_allocation: &backend_uzu::backends::common::Allocation<Metal>,
+ d_allocation: &mut backend_uzu::backends::common::Allocation<Metal>,
) -> Result<f64, BenchError> {
let mut encoder = Encoder::new(context).map_err(|_| BenchError::CommandBuffer)?;
kernel.encode(
- context,
MatmulArguments {
- a: a_buffer,
+ a: a_allocation,
a_offset: 0,
- b: b_buffer,
+ b: b_allocation,
ab_scale: 1.0,
c: MatmulArgumentC::None,
- d: d_buffer,
+ d: d_allocation,
batch_dim: shape.batch as u32,
input_dim: shape.input_dim as u32,
output_dim: shape.output_dim as u32,
@@ -79,35 +81,31 @@ fn run_benchmark(
let b_byte_count = shape.output_dim * shape.input_dim * elem_size;
let d_byte_count = shape.batch * shape.output_dim * elem_size;
- let (a_buffer, b_buffer, mut d_buffer) = match (
- context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
- context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
- context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),
- ) {
- (Some(a), Some(b), Some(d)) => (a, b, d),
- _ => return Err(BenchError::BufferAllocation),
- };
+ let mut a_allocation =
+ context.create_allocation(a_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;
+ let mut b_allocation =
+ context.create_allocation(b_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;
+ let mut d_allocation =
+ context.create_allocation(d_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;
- fill_buffer_random(&a_buffer, a_byte_count);
- fill_buffer_random(&b_buffer, b_byte_count);
+ fill_buffer_random(&mut a_allocation, a_byte_count);
+ fill_buffer_random(&mut b_allocation, b_byte_count);
for iteration in 0..WARMUP_ITERATIONS {
- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {
- BenchError::Warmup {
+ encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation).map_err(
+ |source| BenchError::Warmup {
iteration,
source: Box::new(source),
- }
- })?;
+ },
+ )?;
}
let mut gpu_time_total_ms = 0.0;
for iteration in 0..BENCHMARK_ITERATIONS {
- let gpu_ms =
- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {
- BenchError::Benchmark {
- iteration,
- source: Box::new(source),
- }
+ let gpu_ms = encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation)
+ .map_err(|source| BenchError::Benchmark {
+ iteration,
+ source: Box::new(source),
})?;
gpu_time_total_ms += gpu_ms;
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
index 808c07620..e0f99a8a2 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
@@ -1,9 +1,9 @@
#![cfg(metal_backend)]
-use std::{cell::RefCell, rc::Rc};
+#[macro_use]
+#[path = "../../../../common/mod.rs"]
+mod common;
-use bytemuck;
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use ndarray::{Array, Array3, s};
use super::*;
@@ -64,56 +64,34 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
apply_swaps_3d(&mut expected_keys, &swaps);
apply_swaps_3d(&mut expected_values, &swaps);
- let device = &context.device;
-
- let key_buffer = Rc::new(RefCell::new(
- device
- .new_buffer_with_data(
- bytemuck::cast_slice(key_data.as_slice().unwrap()),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
- .expect("Failed to create buffer"),
- ));
-
- let value_buffer = Rc::new(RefCell::new(
- device
- .new_buffer_with_data(
- bytemuck::cast_slice(value_data.as_slice().unwrap()),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
- .expect("Failed to create buffer"),
- ));
-
- let kv_layer_data = KVLayerData::<Metal> {
- key_buffer: key_buffer.clone(),
- key_shape: [seq_len, num_heads, head_dim],
- value_buffer: value_buffer.clone(),
- value_shape: [seq_len, num_heads, head_dim],
- };
+ let mut key_allocation = common::helpers::alloc_allocation_with_data(context, key_data.as_slice().unwrap());
+ let mut value_allocation = common::helpers::alloc_allocation_with_data(context, value_data.as_slice().unwrap());
let mut encoder = Encoder::new(context).unwrap();
- match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut encoder) {
- Ok(_) => {},
- Err(e) => {
- println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);
- return;
- },
+ {
+ let mut kv_layers = [KVLayerData::<Metal> {
+ key_allocation: &mut key_allocation,
+ key_shape: [seq_len, num_heads, head_dim],
+ value_allocation: &mut value_allocation,
+ value_shape: [seq_len, num_heads, head_dim],
+ }];
+ match kv_cache_update.encode(&mut kv_layers, &source_indices, &destination_indices, &mut encoder) {
+ Ok(_) => {},
+ Err(e) => {
+ println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);
+ return;
+ },
+ }
}
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;
- let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;
-
- let total_elems = num_heads * seq_len * head_dim;
-
- let key_result_slice = unsafe { std::slice::from_raw_parts(key_result_ptr, total_elems) };
- let value_result_slice = unsafe { std::slice::from_raw_parts(value_result_ptr, total_elems) };
-
- let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_result_slice.to_vec())
+ let key_values: Vec<f32> = common::helpers::allocation_to_vec(&key_allocation);
+ let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_values)
.expect("Failed to convert key result to ndarray");
- let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_result_slice.to_vec())
+ let value_values: Vec<f32> = common::helpers::allocation_to_vec(&value_allocation);
+ let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_values)
.expect("Failed to convert value result to ndarray");
println!("Original keys tokens 0,14 head 0:");
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
index 8120fecd1..248dd002a 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
@@ -9,11 +9,11 @@ use half::{bf16, f16};
use crate::{
DataType,
backends::common::{
- Backend, Context, DenseBuffer, Encoder,
+ Allocation, AllocationType, Backend, Context, Encoder, allocation_as_bytes,
gpu_types::QuantizationMode,
kernel::quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
- QuantizedMatmulType, tests::common::helpers::alloc_buffer_with_data,
+ QuantizedMatmulType, tests::common::helpers::alloc_allocation_with_data,
},
},
};
@@ -261,23 +261,23 @@ fn generate_test_quant_params(
}
}
-fn buffer_from_f32_slice<B: Backend>(
+fn allocation_from_f32_slice<B: Backend>(
ctx: &B::Context,
dtype: DataType,
values: &[f32],
-) -> B::DenseBuffer {
+) -> Allocation<B> {
match dtype {
DataType::F16 => {
let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();
- alloc_buffer_with_data::<B, f16>(ctx, data.as_slice())
+ alloc_allocation_with_data::<B, f16>(ctx, data.as_slice())
},
DataType::BF16 => {
let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();
- alloc_buffer_with_data::<B, bf16>(ctx, data.as_slice())
+ alloc_allocation_with_data::<B, bf16>(ctx, data.as_slice())
},
- DataType::F32 => alloc_buffer_with_data::<B, f32>(ctx, values),
+ DataType::F32 => alloc_allocation_with_data::<B, f32>(ctx, values),
other => {
- panic!("Unsupported dtype for buffer_from_f32_slice: {:?}", other)
+ panic!("Unsupported dtype for allocation_from_f32_slice: {:?}", other)
},
}
}
@@ -319,15 +319,17 @@ fn execute_quantized_matmul<B: Backend>(
};
let x_quant = quantize_slice(&x_f32, data_type);
- let w_buf = alloc_buffer_with_data::<B, u8>(ctx, &weights_packed);
- let s_buf = buffer_from_f32_slice::<B>(ctx, data_type, &params.scales);
+ let w_buf = alloc_allocation_with_data::<B, u8>(ctx, &weights_packed);
+ let s_buf = allocation_from_f32_slice::<B>(ctx, data_type, &params.scales);
let b_buf = match quantization_type {
- QuantizedMatmulType::ZeroPoint => alloc_buffer_with_data::<B, u8>(ctx, &params.zero_points),
- QuantizedMatmulType::Mlx => buffer_from_f32_slice::<B>(ctx, data_type, &params.biases),
+ QuantizedMatmulType::ZeroPoint => alloc_allocation_with_data::<B, u8>(ctx, &params.zero_points),
+ QuantizedMatmulType::Mlx => allocation_from_f32_slice::<B>(ctx, data_type, &params.biases),
};
- let x_buf = buffer_from_f32_slice::<B>(ctx, data_type, &x_f32);
- let mut y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");
+ let x_buf = allocation_from_f32_slice::<B>(ctx, data_type, &x_f32);
+ let mut y_buf = ctx
+ .create_allocation(batch * output_dim * data_type.size_in_bytes(), AllocationType::Global)
+ .expect("Failed to create allocation");
let kernel = QuantizedMatmulKernelEncodable::<B>::new(
&ctx,
@@ -350,17 +352,17 @@ fn execute_quantized_matmul<B: Backend>(
if iterations > 1 {
for _ in 0..3 {
let args = QuantizedMatmulArguments {
- a_buffer: &x_buf,
+ a: &x_buf,
a_offset: 0,
- b_buffer: &w_buf,
- scales_buffer: &s_buf,
- zero_points_or_biases_buffer: &b_buf,
- output_buffer: &mut y_buf,
+ b: &w_buf,
+ scales: &s_buf,
+ zero_points_or_biases: &b_buf,
+ output: &mut y_buf,
hadamard_factors: None,
batch_dim: batch,
};
let mut encoder = Encoder::new(ctx).unwrap();
- kernel.encode(&mut encoder, args).unwrap();
+ kernel.encode(&mut encoder, args);
encoder.end_encoding().submit().wait_until_completed().unwrap();
}
}
@@ -368,17 +370,17 @@ fn execute_quantized_matmul<B: Backend>(
let start = Instant::now();
for _ in 0..iterations {
let args = QuantizedMatmulArguments {
- a_buffer: &x_buf,
+ a: &x_buf,
a_offset: 0,
- b_buffer: &w_buf,
- scales_buffer: &s_buf,
- zero_points_or_biases_buffer: &b_buf,
- output_buffer: &mut y_buf,
+ b: &w_buf,
+ scales: &s_buf,
+ zero_points_or_biases: &b_buf,
+ output: &mut y_buf,
hadamard_factors: None,
batch_dim: batch,
};
let mut encoder = Encoder::new(ctx).unwrap();
- kernel.encode(&mut encoder, args).unwrap();
+ kernel.encode(&mut encoder, args);
encoder.end_encoding().submit().wait_until_completed().unwrap();
}
let elapsed = start.elapsed();
@@ -402,20 +404,14 @@ fn execute_quantized_matmul<B: Backend>(
let y_out_f32: Vec<f32> = match data_type {
DataType::F16 => {
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f16;
- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
+ let y_out = bytemuck::cast_slice::<u8, f16>(allocation_as_bytes(&y_buf));
y_out.iter().map(|&v| v.to_f32()).collect()
},
DataType::BF16 => {
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const bf16;
- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
+ let y_out = bytemuck::cast_slice::<u8, bf16>(allocation_as_bytes(&y_buf));
y_out.iter().map(|&v| v.to_f32()).collect()
},
- DataType::F32 => {
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f32;
- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
- y_out.to_vec()
- },
+ DataType::F32 => bytemuck::cast_slice::<u8, f32>(allocation_as_bytes(&y_buf)).to_vec(),
other => panic!("Unsupported dtype for validation: {:?}", other),
};
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
index c261ae43e..85ad712af 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
@@ -1,14 +1,7 @@
-use std::{
- mem::size_of,
- ops::{Deref, DerefMut},
-};
-
-use backend_uzu::backends::common::{Buffer, DenseBuffer};
-
use crate::{
- ArrayContextExt, DataType,
+ DataType, allocation_to_vec,
backends::common::{
- Backend, Context, Encoder, Kernels,
+ Allocation, Backend, Context, Encoder, Kernels,
gpu_types::ActivationType,
kernel::{
Conv1dScanKernel,
@@ -21,30 +14,11 @@ use crate::{
#[path = "../../../../common/mod.rs"]
mod common;
-fn write_buffer<B: Backend>(
- buf: &B::DenseBuffer,
- data: &[f32],
-) {
- unsafe {
- std::ptr::copy_nonoverlapping(data.as_ptr(), buf.cpu_ptr().as_ptr() as *mut f32, data.len());
- }
-}
-
-fn read_buffer<B: Backend>(
- buf: &B::DenseBuffer,
- len: usize,
-) -> Vec<f32> {
- let mut out = vec![0.0f32; len];
- unsafe {
- std::ptr::copy_nonoverlapping(buf.cpu_ptr().as_ptr() as *const f32, out.as_mut_ptr(), len);
- }
- out
-}
-
-fn zero_buffer<B: Backend>(buf: &B::DenseBuffer) {
- unsafe {
- std::ptr::write_bytes(buf.cpu_ptr().as_ptr(), 0, buf.size());
- }
+fn allocation_from_slice<B: Backend, T: crate::ArrayElement>(
+ context: &B::Context,
+ data: &[T],
+) -> Allocation<B> {
+ common::helpers::alloc_allocation_with_data(context, data)
}
fn ssd_prefill_cpu_reference(
@@ -105,7 +79,6 @@ fn ssd_prefill_cpu_reference(
(y_out, state)
}
-#[allow(dead_code)]
struct SSDPrefillFixture {
suffix_len: usize,
num_heads: usize,
@@ -113,9 +86,6 @@ struct SSDPrefillFixture {
state_dim: usize,
group_size: i32,
total_x: usize,
- total_dt: usize,
- total_cb: usize,
- total_state: usize,
x_strides: [usize; 3],
dt_strides: [usize; 2],
cb_strides: [usize; 3],
@@ -163,9 +133,6 @@ impl SSDPrefillFixture {
state_dim,
group_size,
total_x,
- total_dt,
- total_cb,
- total_state,
x_strides,
dt_strides,
cb_strides,
@@ -187,56 +154,24 @@ fn run_prefill_kernel_mode<B: Backend>(
fixture: &SSDPrefillFixture,
mode: SSDPrefillMode,
) -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {
- let x_array = ctx.create_array_from(&[fixture.x_data.len()], &fixture.x_data, "");
- let x_array_buf = x_array.buffer();
- let x_array_buf_borrow = x_array_buf.borrow();
- let x_buf = x_array_buf_borrow.deref();
-
- let dt_array = ctx.create_array_from(&[fixture.dt_data.len()], &fixture.dt_data, "");
- let dt_array_buf = dt_array.buffer();
- let dt_array_buf_borrow = dt_array_buf.borrow();
- let dt_buf = dt_array_buf_borrow.deref();
-
- let b_array = ctx.create_array_from(&[fixture.b_data.len()], &fixture.b_data, "");
- let b_array_buf = b_array.buffer();
- let b_array_buf_borrow = b_array_buf.borrow();
- let b_buf = b_array_buf_borrow.deref();
-
- let c_array = ctx.create_array_from(&[fixture.c_data.len()], &fixture.c_data, "");
- let c_array_buf = c_array.buffer();
- let c_array_buf_borrow = c_array_buf.borrow();
- let c_buf = c_array_buf_borrow.deref();
-
- let d_array = ctx.create_array_from(&[fixture.d_data.len()], &fixture.d_data, "");
- let d_array_buf = d_array.buffer();
- let d_array_buf_borrow = d_array_buf.borrow();
- let d_buf = d_array_buf_borrow.deref();
-
- let z_array = ctx.create_array_from(&[fixture.z_data.len()], &fixture.z_data, "");
- let z_array_buf = z_array.buffer();
- let z_array_buf_borrow = z_array_buf.borrow();
- let z_buf = z_array_buf_borrow.deref();
-
- let state_array = ctx.create_array_from(&[fixture.state_init.len()], &fixture.state_init, "");
- let state_array_buf = state_array.buffer();
- let mut state_array_buf_borrow = state_array_buf.borrow_mut();
- let state_buf = state_array_buf_borrow.deref_mut();
-
- let y = vec![0f32; fixture.total_x];
- let y_array = ctx.create_array_from(&[y.len()], &y, "");
- let y_array_buf = y_array.buffer();
- let mut y_array_buf_borrow = y_array_buf.borrow_mut();
- let y_buf = y_array_buf_borrow.deref_mut();
-
- let args = SSDPrefillArguments::<B> {
- x: x_buf,
- dt: dt_buf,
- b: b_buf,
- c: c_buf,
- d: d_buf,
- z: z_buf,
- state: state_buf,
- y: y_buf,
+ let x_buf = allocation_from_slice::<B, _>(ctx, &fixture.x_data);
+ let dt_buf = allocation_from_slice::<B, _>(ctx, &fixture.dt_data);
+ let b_buf = allocation_from_slice::<B, _>(ctx, &fixture.b_data);
+ let c_buf = allocation_from_slice::<B, _>(ctx, &fixture.c_data);
+ let d_buf = allocation_from_slice::<B, _>(ctx, &fixture.d_data);
+ let z_buf = allocation_from_slice::<B, _>(ctx, &fixture.z_data);
+ let mut state_buf = allocation_from_slice::<B, _>(ctx, &fixture.state_init);
+ let mut y_buf = allocation_from_slice::<B, _>(ctx, &vec![0f32; fixture.total_x]);
+
+ let args = SSDPrefillArguments {
+ x: &x_buf,
+ dt: &dt_buf,
+ b: &b_buf,
+ c: &c_buf,
+ d: &d_buf,
+ z: &z_buf,
+ state: &mut state_buf,
+ y: &mut y_buf,
suffix_len: fixture.suffix_len,
group_size: fixture.group_size as u32,
state_size: fixture.state_dim as u32,
@@ -252,8 +187,8 @@ fn run_prefill_kernel_mode<B: Backend>(
kernel.encode(&mut encoder, args, mode);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let y_vec = read_buffer::<B>(y_buf, fixture.total_x);
- let state_vec = read_buffer::<B>(state_buf, fixture.total_state);
+ let y_vec = allocation_to_vec::<B, f32>(&y_buf);
+ let state_vec = allocation_to_vec::<B, f32>(&state_buf);
(y_vec, state_vec, None)
}
@@ -275,106 +210,61 @@ fn run_conv_scan_once<B: Backend>(
let _total_w = channels * kernel_size as usize;
let total_state = channels * tap_count;
- let y_array = if alias_io {
- ctx.create_array_from(&[x_data.len()], x_data, "")
+ let mut y_buf = if alias_io {
+ allocation_from_slice::<B, _>(ctx, x_data)
} else {
- ctx.create_array_zeros(&[total_x], DataType::F32, "")
+ allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_x])
};
- let b_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");
- let c_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");
-
- let w_array = ctx.create_array_from(&[w_data.len()], w_data, "");
- let b_array = ctx.create_array_from(&[b_data.len()], b_data, "");
-
- let state_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");
- let scratch_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");
-
- {
- let state_buf = state_array.buffer();
- let state_borrow = state_buf.borrow();
- write_buffer::<B>(state_borrow.deref(), state_init);
- if use_scratch && tap_count > 0 {
- let scratch_buf = scratch_array.buffer();
- let scratch_borrow = scratch_buf.borrow();
- zero_buffer::<B>(scratch_borrow.deref());
- }
- }
+ let mut b_out_buf = allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_x]);
+ let mut c_out_buf = allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_x]);
+ let w_buf = allocation_from_slice::<B, _>(ctx, w_data);
+ let b_buf = allocation_from_slice::<B, _>(ctx, b_data);
+ let mut state_buf = allocation_from_slice::<B, _>(ctx, state_init);
+ let scratch_buf = allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_state]);
let padded_len = tap_count + suffix_len;
- let padded_array = ctx.create_array_uninitialized(&[padded_len * channels], DataType::F32, "");
- {
- let padded_buf = padded_array.buffer();
- let padded_borrow = padded_buf.borrow();
- let mut host = vec![0.0f32; padded_len * channels];
- for tap in 0..tap_count {
- for ch in 0..channels {
- host[tap * channels + ch] = state_init[ch * tap_count + tap];
- }
+ let mut padded_host = vec![0.0f32; padded_len * channels];
+ for tap in 0..tap_count {
+ for ch in 0..channels {
+ padded_host[tap * channels + ch] = state_init[ch * tap_count + tap];
}
- for token in 0..suffix_len {
- for ch in 0..channels {
- host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];
- }
+ }
+ for token in 0..suffix_len {
+ for ch in 0..channels {
+ padded_host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];
}
- write_buffer::<B>(padded_borrow.deref(), &host);
}
+ let padded_buf = allocation_from_slice::<B, _>(ctx, &padded_host);
let mut encoder = Encoder::new(ctx).unwrap();
- {
- let padded_buf = padded_array.buffer();
- let padded_borrow = padded_buf.borrow();
- let w_buf = w_array.buffer();
- let w_borrow = w_buf.borrow();
- let b_buf = b_array.buffer();
- let b_borrow = b_buf.borrow();
- let y_buf = y_array.buffer();
- let mut y_borrow = y_buf.borrow_mut();
- let b_out_buf = b_out_array.buffer();
- let mut b_out_borrow = b_out_buf.borrow_mut();
- let c_out_buf = c_out_array.buffer();
- let mut c_out_borrow = c_out_buf.borrow_mut();
- let state_buf = state_array.buffer();
- let mut state_borrow = state_buf.borrow_mut();
-
- kernel.encode(
- padded_borrow.deref(),
- w_borrow.deref(),
- Some(b_borrow.deref()),
- y_borrow.deref_mut(),
- b_out_borrow.deref_mut(),
- c_out_borrow.deref_mut(),
- state_borrow.deref_mut(),
- suffix_len as u32,
- kernel_size as u32,
- channels as u32,
- tap_count as u32,
- channels as u32,
- channels as u32,
- 0u32,
- ActivationType::SILU,
- &mut encoder,
- );
- }
+ kernel.encode(
+ &padded_buf,
+ &w_buf,
+ Some(&b_buf),
+ &mut y_buf,
+ &mut b_out_buf,
+ &mut c_out_buf,
+ &mut state_buf,
+ suffix_len as u32,
+ kernel_size as u32,
+ channels as u32,
+ tap_count as u32,
+ channels as u32,
+ channels as u32,
+ 0u32,
+ ActivationType::SILU,
+ &mut encoder,
+ );
if use_scratch && tap_count > 0 {
let bytes = channels * tap_count * size_of::<f32>();
- if bytes > 0 {
- let scratch_buf = scratch_array.buffer();
- let scratch_borrow = scratch_buf.borrow();
- let state_buf = state_array.buffer();
- let mut state_borrow = state_buf.borrow_mut();
- encoder.encode_copy(scratch_borrow.deref(), 0..bytes, state_borrow.deref_mut(), 0..bytes);
- }
+ encoder.encode_copy(&scratch_buf, 0..bytes, &mut state_buf, 0..bytes);
}
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let y_buf = y_array.buffer();
- let y_borrow = y_buf.borrow();
- let state_buf = state_array.buffer();
- let state_borrow = state_buf.borrow();
- let y_vec = read_buffer::<B>(y_borrow.deref(), total_x);
- let state_vec = read_buffer::<B>(state_borrow.deref(), total_state);
+ let y_vec = allocation_to_vec::<B, f32>(&y_buf);
+ let state_vec = allocation_to_vec::<B, f32>(&state_buf);
(y_vec, state_vec)
}
diff --git a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs
index b1c7a5e47..0122377f3 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs
@@ -1,18 +1,13 @@
#![cfg(metal_backend)]
-use std::mem::size_of;
-
-use bytemuck;
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use ndarray::{Array4, s};
-use objc2::{rc::Retained, runtime::ProtocolObject};
use test_tag::tag;
use crate::{
DataType,
backends::{
common::{
- Backend, Context, Encoder, Kernels,
+ Allocation, Backend, Context, Encoder, Kernels,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
@@ -22,227 +17,199 @@ use crate::{
},
};
+#[path = "../../common/mod.rs"]
+mod common;
+
+use common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder};
+
fn reference_attention(
- queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]
- keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
- values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]
+ queries: &Array4<f32>,
+ keys: &Array4<f32>,
+ values: &Array4<f32>,
sinks: Option<&[f32]>,
scale: f32,
is_causal: bool,
) -> Array4<f32> {
- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
- let (_, num_kv_heads, kv_seq_len, _) = keys.dim();
- let n_repeats = num_heads / num_kv_heads;
-
- let scaled_queries = queries.mapv(|x| x * scale);
-
- let mut output = Array4::zeros((batch_size, num_heads, seq_len, head_dim));
-
- for b in 0..batch_size {
- if n_repeats > 1 {
- for kv_head in 0..num_kv_heads {
- for repeat in 0..n_repeats {
- let q_head = kv_head * n_repeats + repeat;
-
- let q = scaled_queries.slice(s![b, q_head, .., ..]).to_owned(); // [L, D]
- let k = keys.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]
- let v = values.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]
-
- // Efficient matrix multiplication: scores = Q @ K^T
- let mut scores = q.dot(&k.t());
-
- if is_causal {
- for i in 0..seq_len {
- for j in (i + 1)..kv_seq_len {
- scores[[i, j]] = f32::NEG_INFINITY;
- }
- }
- }
-
- // Softmax with optional sink logit
- for i in 0..seq_len {
- let sink_logit = sinks.map(|s| s[q_head]);
- let row = scores.row(i);
- let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));
-
- let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);
- for j in 0..kv_seq_len {
- scores[[i, j]] = (scores[[i, j]] - max_score).exp();
- sum_exp += scores[[i, j]];
- }
- scores.row_mut(i).mapv_inplace(|x| x / sum_exp);
- }
-
- // Efficient matrix multiplication: output = scores @ V
- let head_output = scores.dot(&v);
- output.slice_mut(s![b, q_head, .., ..]).assign(&head_output);
+ let (batch_size, num_heads, sequence_length, head_dim) = queries.dim();
+ let (_, num_kv_heads, kv_sequence_length, _) = keys.dim();
+ let repeats_per_kv_head = num_heads / num_kv_heads;
+ let scaled_queries = queries.mapv(|value| value * scale);
+ let mut output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));
+
+ for batch_index in 0..batch_size {
+ if repeats_per_kv_head > 1 {
+ for kv_head_index in 0..num_kv_heads {
+ for repeat_index in 0..repeats_per_kv_head {
+ let head_index = kv_head_index * repeats_per_kv_head + repeat_index;
+ assign_attention_head_output(
+ &mut output,
+ &scaled_queries,
+ keys,
+ values,
+ sinks,
+ is_causal,
+ batch_index,
+ head_index,
+ kv_head_index,
+ sequence_length,
+ kv_sequence_length,
+ );
}
}
} else {
- for h in 0..num_heads {
- let q = scaled_queries.slice(s![b, h, .., ..]).to_owned(); // [L, D]
- let k = keys.slice(s![b, h, .., ..]).to_owned(); // [L_kv, D]
- let v = values.slice(s![b, h, .., ..]).to_owned(); // [L_kv, D]
-
- // Efficient matrix multiplication: scores = Q @ K^T
- let mut scores = q.dot(&k.t());
-
- if is_causal {
- for i in 0..seq_len {
- for j in (i + 1)..kv_seq_len {
- scores[[i, j]] = f32::NEG_INFINITY;
- }
- }
- }
+ for head_index in 0..num_heads {
+ assign_attention_head_output(
+ &mut output,
+ &scaled_queries,
+ keys,
+ values,
+ sinks,
+ is_causal,
+ batch_index,
+ head_index,
+ head_index,
+ sequence_length,
+ kv_sequence_length,
+ );
+ }
+ }
+ }
- // Softmax with optional sink logit
- for i in 0..seq_len {
- let sink_logit = sinks.map(|s| s[h]);
- let row = scores.row(i);
- let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));
-
- let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);
- for j in 0..kv_seq_len {
- scores[[i, j]] = (scores[[i, j]] - max_score).exp();
- sum_exp += scores[[i, j]];
- }
- scores.row_mut(i).mapv_inplace(|x| x / sum_exp);
- }
+ output
+}
- // Efficient matrix multiplication: output = scores @ V
- let head_output = scores.dot(&v);
- output.slice_mut(s![b, h, .., ..]).assign(&head_output);
+fn assign_attention_head_output(
+ output: &mut Array4<f32>,
+ scaled_queries: &Array4<f32>,
+ keys: &Array4<f32>,
+ values: &Array4<f32>,
+ sinks: Option<&[f32]>,
+ is_causal: bool,
+ batch_index: usize,
+ head_index: usize,
+ kv_head_index: usize,
+ sequence_length: usize,
+ kv_sequence_length: usize,
+) {
+ let queries = scaled_queries.slice(s![batch_index, head_index, .., ..]).to_owned();
+ let keys = keys.slice(s![batch_index, kv_head_index, .., ..]).to_owned();
+ let values = values.slice(s![batch_index, kv_head_index, .., ..]).to_owned();
+ let mut scores = queries.dot(&keys.t());
+
+ if is_causal {
+ for query_index in 0..sequence_length {
+ for key_index in (query_index + 1)..kv_sequence_length {
+ scores[[query_index, key_index]] = f32::NEG_INFINITY;
}
}
}
- output
+ for query_index in 0..sequence_length {
+ let sink_logit = sinks.map(|sink_values| sink_values[head_index]);
+ let row = scores.row(query_index);
+ let max_score =
+ row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |current_max, &score| current_max.max(score));
+
+ let mut sum_exp = sink_logit.map(|sink_value| (sink_value - max_score).exp()).unwrap_or(0.0);
+ for key_index in 0..kv_sequence_length {
+ scores[[query_index, key_index]] = (scores[[query_index, key_index]] - max_score).exp();
+ sum_exp += scores[[query_index, key_index]];
+ }
+ scores.row_mut(query_index).mapv_inplace(|value| value / sum_exp);
+ }
+
+ let head_output = scores.dot(&values);
+ output.slice_mut(s![batch_index, head_index, .., ..]).assign(&head_output);
}
fn create_test_data(
batch_size: usize,
num_heads: usize,
num_kv_heads: usize,
- seq_len: usize,
+ sequence_length: usize,
head_dim: usize,
seed: u64,
) -> (Array4<f32>, Array4<f32>, Array4<f32>) {
use rand::{RngExt, SeedableRng, rngs::StdRng};
- let mut rng = StdRng::seed_from_u64(seed);
-
- let queries = Array4::from_shape_fn((batch_size, num_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));
-
- let keys = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));
+ let mut random = StdRng::seed_from_u64(seed);
- let values = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));
+ let queries =
+ Array4::from_shape_fn((batch_size, num_heads, sequence_length, head_dim), |_| random.random_range(-0.5..0.5));
+ let keys = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {
+ random.random_range(-0.5..0.5)
+ });
+ let values = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {
+ random.random_range(-0.5..0.5)
+ });
(queries, keys, values)
}
-/// Convert ndarray to Metal buffer layout expected by our kernel
-fn create_query_buffer(
+fn create_query_allocation(
queries: &Array4<f32>,
context: &<Metal as Backend>::Context,
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
- let (_batch_size, num_heads, seq_len, head_dim) = queries.dim();
-
- // Our kernel expects queries layout: [num_heads, seq_len, head_dim]
- let mut query_data = vec![0.0f32; num_heads * seq_len * head_dim];
-
- for h in 0..num_heads {
- for t in 0..seq_len {
- for d in 0..head_dim {
- let idx = h * seq_len * head_dim + t * head_dim + d;
- query_data[idx] = queries[[0, h, t, d]];
+) -> Allocation<Metal> {
+ let (_batch_size, num_heads, sequence_length, head_dim) = queries.dim();
+ let mut values = vec![0.0_f32; num_heads * sequence_length * head_dim];
+
+ for head_index in 0..num_heads {
+ for sequence_index in 0..sequence_length {
+ for dim_index in 0..head_dim {
+ let flat_index = head_index * sequence_length * head_dim + sequence_index * head_dim + dim_index;
+ values[flat_index] = queries[[0, head_index, sequence_index, dim_index]];
}
}
}
- context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&query_data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
+ alloc_allocation_with_data(context, &values)
}
-fn create_key_cache_buffer(
- keys: &Array4<f32>,
- max_seq_len: usize,
- context: &<Metal as Backend>::Context,
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
- let (_batch_size, num_kv_heads, seq_len, head_dim) = keys.dim();
-
- // Our kernel expects key cache layout: [max_seq_len, num_kv_heads, head_dim]
- let mut key_cache_data = vec![0.0f32; max_seq_len * num_kv_heads * head_dim];
-
- for t in 0..seq_len {
- for h in 0..num_kv_heads {
- for d in 0..head_dim {
- let idx = t * num_kv_heads * head_dim + h * head_dim + d;
- key_cache_data[idx] = keys[[0, h, t, d]];
- }
- }
- }
-
- context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&key_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
-}
-
-fn create_value_cache_buffer(
+fn create_attention_cache_allocation(
values: &Array4<f32>,
- max_seq_len: usize,
+ max_sequence_length: usize,
context: &<Metal as Backend>::Context,
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
- let (_batch_size, num_kv_heads, seq_len, head_dim) = values.dim();
-
- // Our kernel expects value cache layout: [max_seq_len, num_kv_heads, head_dim]
- let mut value_cache_data = vec![0.0f32; max_seq_len * num_kv_heads * head_dim];
-
- for t in 0..seq_len {
- for h in 0..num_kv_heads {
- for d in 0..head_dim {
- let idx = t * num_kv_heads * head_dim + h * head_dim + d;
- value_cache_data[idx] = values[[0, h, t, d]];
+) -> Allocation<Metal> {
+ let (_batch_size, num_kv_heads, sequence_length, head_dim) = values.dim();
+ let mut cache = vec![0.0_f32; max_sequence_length * num_kv_heads * head_dim];
+
+ for sequence_index in 0..sequence_length {
+ for head_index in 0..num_kv_heads {
+ for dim_index in 0..head_dim {
+ let flat_index = sequence_index * num_kv_heads * head_dim + head_index * head_dim + dim_index;
+ cache[flat_index] = values[[0, head_index, sequence_index, dim_index]];
}
}
}
- context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&value_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
+ alloc_allocation_with_data(context, &cache)
}
-fn create_sinks_buffer(
+fn create_sinks_allocation(
sinks: &[f32],
context: &<Metal as Backend>::Context,
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
- context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&sinks), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
+) -> Allocation<Metal> {
+ alloc_allocation_with_data(context, sinks)
}
fn convert_kernel_output(
- output_slice: &[f32],
+ output: &[f32],
batch_size: usize,
num_heads: usize,
- seq_len: usize,
+ sequence_length: usize,
head_dim: usize,
) -> Array4<f32> {
- let mut kernel_output = Array4::zeros((batch_size, num_heads, seq_len, head_dim));
- for h in 0..num_heads {
- for t in 0..seq_len {
- for d in 0..head_dim {
- // Metal kernel output layout: o_offset = q_seq_idx * num_heads + head_idx
- let o_offset = t * num_heads + h;
- let idx = o_offset * head_dim + d;
- kernel_output[[0, h, t, d]] = output_slice[idx];
+ let mut kernel_output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));
+
+ for head_index in 0..num_heads {
+ for sequence_index in 0..sequence_length {
+ for dim_index in 0..head_dim {
+ let flat_index = (sequence_index * num_heads + head_index) * head_dim + dim_index;
+ kernel_output[[0, head_index, sequence_index, dim_index]] = output[flat_index];
}
}
}
+
kernel_output
}
@@ -258,16 +225,11 @@ fn run_single_pass_attention(
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
- let query_buffer = create_query_buffer(queries, context);
- let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
- let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
-
- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
-
- let mut output_buffer = context
- .device
- .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let query_buffer = create_query_allocation(queries, context);
+ let key_cache_buffer = create_attention_cache_allocation(keys, seq_len, context);
+ let value_cache_buffer = create_attention_cache_allocation(values, seq_len, context);
+ let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));
+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
@@ -284,77 +246,63 @@ fn run_single_pass_attention(
(num_kv_heads * head_dim) as u32,
None,
scale,
- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
+ None::<&Allocation<Metal>>,
None,
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
&mut encoder,
);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
+ submit_encoder(encoder);
- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
+ let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);
+ let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);
Ok(kernel_output)
}
-fn run_single_pass_attention_with_is_causal(
- kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,
+fn create_single_pass_kernel(
context: &<Metal as Backend>::Context,
- queries: &Array4<f32>,
- keys: &Array4<f32>,
- values: &Array4<f32>,
- sinks: Option<&[f32]>,
- scale: f32,
- _is_causal: bool,
-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {
- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
- let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
-
- let query_buffer = create_query_buffer(queries, context);
- let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
- let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
-
- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
-
- let mut output_buffer = context
- .device
- .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut encoder = Encoder::new(context).expect("Failed to create encoder");
-
- kernel.encode(
- &query_buffer,
- &key_cache_buffer,
- &value_cache_buffer,
- &mut output_buffer,
- (num_heads / num_kv_heads) as u32,
- seq_len as u32,
- head_dim as u32,
- (num_kv_heads * head_dim) as u32,
+ head_dim: usize,
+ has_sinks: bool,
+ is_causal: bool,
+) -> <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel {
+ <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
+ context,
+ DataType::F32,
head_dim as u32,
- (num_kv_heads * head_dim) as u32,
- None,
- scale,
- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
- None,
- sinks_buffer.as_ref().map(|b| b),
- num_heads as u32,
- seq_len as u32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
-
- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
+ has_sinks,
+ false,
+ is_causal,
+ false,
+ false,
+ )
+ .expect("Failed to create attention single pass kernel")
+}
- Ok(kernel_output)
+fn create_two_pass_kernels(
+ context: &<Metal as Backend>::Context,
+ head_dim: usize,
+ is_causal: bool,
+) -> (
+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,
+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,
+) {
+ (
+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
+ context,
+ DataType::F32,
+ head_dim as u32,
+ false,
+ false,
+ is_causal,
+ false,
+ false,
+ )
+ .expect("Failed to create AttentionTwoPass1Kernel"),
+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, DataType::F32, head_dim as u32)
+ .expect("Failed to create AttentionTwoPass2Kernel"),
+ )
}
fn run_gemm_attention(
@@ -370,26 +318,23 @@ fn run_gemm_attention(
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();
- let query_buffer = create_query_buffer(queries, context);
- let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);
- let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);
+ let query_allocation = create_query_allocation(queries, context);
+ let key_allocation = create_attention_cache_allocation(keys, seq_len, context);
+ let value_allocation = create_attention_cache_allocation(values, seq_len, context);
- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
+ let sinks_allocation = sinks.map(|sinks| create_sinks_allocation(sinks, context));
- let mut output_buffer = context
- .device
- .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut output_allocation = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
let args = AttentionGemmArguments {
- queries_buffer: &query_buffer,
- keys_buffer: &key_cache_buffer,
- values_buffer: &value_cache_buffer,
- output_buffer: &mut output_buffer,
- trie_buffer: None,
- sinks_buffer: sinks_buffer.as_ref(),
+ queries: &query_allocation,
+ keys: &key_allocation,
+ values: &value_allocation,
+ output: &mut output_allocation,
+ trie: None,
+ sinks: sinks_allocation.as_ref(),
num_heads,
num_groups: num_kv_heads,
suffix_length: seq_len,
@@ -407,13 +352,12 @@ fn run_gemm_attention(
v_seq_stride: (num_kv_heads * head_dim) as u64,
};
- kernel.encode(context, &mut encoder, args)?;
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ kernel.encode(&mut encoder, args)?;
+ submit_encoder(encoder);
- let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
+ let output: Vec<f32> = allocation_to_vec(&output_allocation);
- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
+ let kernel_output = convert_kernel_output(&output, batch_size, num_heads, seq_len, head_dim);
Ok(kernel_output)
}
@@ -424,36 +368,8 @@ fn compare_results(
tolerance: f32,
test_name: &str,
) -> Result<(), String> {
- let max_diff = kernel_output.iter().zip(reference_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);
-
- let relative_error = kernel_output
- .iter()
- .zip(reference_output.iter())
- .map(|(a, b)| {
- if b.abs() > 1e-6 {
- (a - b).abs() / b.abs()
- } else {
- (a - b).abs()
- }
- })
- .fold(0.0f32, f32::max);
-
- println!("Max absolute difference ({}): {}", test_name, max_diff);
- println!("Max relative error ({}): {}", test_name, relative_error);
-
- println!("Debug - Sample values ({}):", test_name);
- println!("Kernel output [0,0,0,0:4]: {:?}", &kernel_output.slice(s![0, 0, 0, 0..4]).to_vec());
- println!("Reference output [0,0,0,0:4]: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
- println!("Kernel output [0,0,1,0:4]: {:?}", &kernel_output.slice(s![0, 0, 1, 0..4]).to_vec());
- println!("Reference output [0,0,1,0:4]: {:?}", &reference_output.slice(s![0, 0, 1, 0..4]).to_vec());
-
- let kernel_max = kernel_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));
- let kernel_min = kernel_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
- let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));
- let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
-
- println!("Kernel output range: [{}, {}]", kernel_min, kernel_max);
- println!("Reference output range: [{}, {}]", ref_min, ref_max);
+ let max_diff =
+ kernel_output.iter().zip(reference_output.iter()).map(|(lhs, rhs)| (lhs - rhs).abs()).fold(0.0_f32, f32::max);
if max_diff >= tolerance {
return Err(format!(
@@ -462,7 +378,6 @@ fn compare_results(
));
}
- println!("✓ {} test passed!", test_name);
Ok(())
}
@@ -478,39 +393,12 @@ fn test_single_pass_attention_basic() {
let scale = 1.0 / (head_dim as f32).sqrt();
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
-
- println!("Testing reference implementation...");
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
+ let kernel = create_single_pass_kernel(&context, head_dim, false, false);
+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
+ .expect("single-pass attention");
- let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));
- let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));
- println!("Reference output range: [{}, {}]", ref_min, ref_max);
- println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
-
- let is_causal = false; // Non-causal attention for this test
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- false,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create attention single pass metal");
-
- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run single-pass attention: {:?}", e);
- },
- };
-
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention") {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention").unwrap();
}
#[test]
@@ -527,17 +415,11 @@ fn test_gemm_attention_basic() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
let kernel = AttentionGemmBlock::new(DataType::F32);
-
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
+ let kernel_output = run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, false)
+ .expect("run gemm attention");
- let kernel_output =
- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false)
- .expect("run gemm attention");
-
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention") {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention").unwrap();
}
#[test]
@@ -554,17 +436,11 @@ fn test_gemm_attention_f32_head_dim_128() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);
let kernel = AttentionGemmBlock::new(DataType::F32);
-
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
+ let kernel_output = run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, false)
+ .expect("run gemm attention f32 head_dim=128");
- let kernel_output =
- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false)
- .expect("run gemm attention f32 head_dim=128");
-
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention f32 head_dim=128") {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention f32 head_dim=128").unwrap();
}
#[test]
@@ -581,31 +457,13 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
- // CPU reference
let reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);
-
- // vector attention path (single-pass)
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- false,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create attention single pass metal");
- let vector_output =
- run_single_pass_attention_with_is_causal(&kernel, &context, &queries, &keys, &values, None, scale, is_causal)
- .expect("run vector attention");
-
- // matrix attention path (gemm)
+ let kernel = create_single_pass_kernel(&context, head_dim, false, is_causal);
+ let vector_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
+ .expect("run vector attention");
let gemm_kernel = AttentionGemmBlock::new(DataType::F32);
let matrix_output = run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, None, scale, is_causal)
.expect("run matrix attention");
-
- // Compare both to CPU
let tol_cpu = 5e-2;
compare_results(&vector_output, &reference_output, tol_cpu, "vector single-pass attention vs CPU").unwrap();
compare_results(&matrix_output, &reference_output, tol_cpu, "matrix attention vs CPU").unwrap();
@@ -613,7 +471,6 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let max_diff_vector_matrix =
vector_output.iter().zip(matrix_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);
let tol_vector_matrix = 5e-2;
- println!("Max absolute difference (vector single-pass attention vs matrix attention): {}", max_diff_vector_matrix);
assert!(
max_diff_vector_matrix <= tol_vector_matrix,
"vector single-pass attention and matrix attention differ too much: max_diff = {} (tol={})",
@@ -636,34 +493,13 @@ fn test_single_pass_attention_with_sinks() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
- println!("Using sinks: {:?}", sinks);
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- true,
- false,
- false,
- false,
- false,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
+ let kernel = create_single_pass_kernel(&context, head_dim, true, false);
let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)
+ .expect("single-pass attention with sinks");
- let kernel_output =
- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run single-pass attention with sinks: {:?}", e);
- },
- };
-
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with sinks") {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention with sinks").unwrap();
}
#[test]
@@ -680,39 +516,14 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
- println!("Long sequence sinks: {:?}", sinks);
-
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- true,
- false,
- false,
- false,
- false,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
+ let kernel = create_single_pass_kernel(&context, head_dim, true, false);
let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);
+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)
+ .expect("single-pass attention with sinks (long sequence)");
- let kernel_output =
- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run single-pass attention long sequence with sinks: {:?}", e);
- },
- };
-
- let tolerance = 5e-2;
- if let Err(e) = compare_results(
- &kernel_output,
- &reference_output,
- tolerance,
- "Single-pass attention with sinks (long sequence)",
- ) {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 5e-2, "Single-pass attention with sinks (long sequence)")
+ .unwrap();
}
#[test]
@@ -728,32 +539,12 @@ fn test_single_pass_attention_gqa() {
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let is_causal = false; // Non-causal attention for this test
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- false,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");
-
- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run single-pass attention GQA: {:?}", e);
- },
- };
+ let kernel = create_single_pass_kernel(&context, head_dim, false, false);
+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)
+ .expect("single-pass attention GQA");
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
-
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention GQA") {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention GQA").unwrap();
}
fn run_two_pass_attention(
@@ -769,37 +560,19 @@ fn run_two_pass_attention(
let (batch_size, num_heads, seq_len, head_dim) = queries.dim();
let (_, num_kv_heads, _, _) = keys.dim();
- let queries_buffer = create_query_buffer(queries, context);
- let keys_buffer = create_key_cache_buffer(keys, seq_len, context);
- let values_buffer = create_value_cache_buffer(values, seq_len, context);
- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));
+ let queries_buffer = create_query_allocation(queries, context);
+ let keys_buffer = create_attention_cache_allocation(keys, seq_len, context);
+ let values_buffer = create_attention_cache_allocation(values, seq_len, context);
+ let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));
let total_blocks_count = 32;
let partials_size = num_heads * seq_len * total_blocks_count * head_dim;
let sums_maxs_size = num_heads * seq_len * total_blocks_count;
- let mut partials_buffer = context
- .device
- .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut sums_buffer = context
- .device
- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut maxs_buffer = context
- .device
- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut output_buffer = context
- .device
- .new_buffer(
- num_heads * seq_len * head_dim * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
- .expect("Failed to create buffer");
+ let mut partials_buffer = alloc_allocation::<Metal, f32>(context, partials_size);
+ let mut sums_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);
+ let mut maxs_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);
+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);
let mut encoder = Encoder::new(context).expect("Failed to create encoder");
@@ -820,7 +593,7 @@ fn run_two_pass_attention(
scale,
num_heads as u32,
seq_len as u32,
- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
+ None::<&Allocation<Metal>>,
None,
sinks_buffer.as_ref().map(|b| b),
&mut encoder,
@@ -834,12 +607,10 @@ fn run_two_pass_attention(
seq_len as u32,
&mut encoder,
);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
+ submit_encoder(encoder);
- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);
+ let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);
+ let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);
Ok(kernel_output)
}
@@ -860,35 +631,12 @@ fn test_two_pass_attention() {
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- false,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
- let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
+ let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);
let kernel_output =
- match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run two-pass attention: {:?}", e);
- },
- };
+ run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)
+ .expect("two-pass attention");
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention") {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention").unwrap();
}
#[test]
@@ -907,35 +655,12 @@ fn test_two_pass_attention_gqa() {
let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);
- let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- false,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");
- let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- )
- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");
+ let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);
let kernel_output =
- match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {
- Ok(output) => output,
- Err(e) => {
- panic!("Failed to run two-pass attention GQA: {:?}", e);
- },
- };
+ run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)
+ .expect("two-pass attention GQA");
- let tolerance = 1e-2;
- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention GQA") {
- panic!("{}", e);
- }
+ compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention GQA").unwrap();
}
#[tag(heavy)]
@@ -955,66 +680,27 @@ fn perf_two_pass_attention() {
let scale = 1.0 / (head_dim as f32).sqrt();
let is_causal = false;
- println!(
- "Creating test data for two-pass performance test (prefix={}, suffix={})...",
- seq_len - suffix_length,
- suffix_length
- );
let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
+ let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);
- let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- false,
- false,
- is_causal,
- false,
- false,
- )
- .expect("Failed to create AttentionTwoPass1Kernel");
- let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(
- &context,
- DataType::F32,
- head_dim as u32,
- )
- .expect("Failed to create AttentionTwoPass2Kernel");
-
- // ---- Create buffers ----
- // For realistic inference, we only process queries for the suffix (new tokens)
- let queries_suffix = queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();
- let queries_buffer = create_query_buffer(&queries_suffix, &context);
- let keys_buffer = create_key_cache_buffer(&keys, seq_len, &context);
- let values_buffer = create_value_cache_buffer(&values, seq_len, &context);
+ let suffix_start = (seq_len - suffix_length) as usize;
+ let queries_suffix = queries.slice(s![.., .., suffix_start.., ..]).to_owned();
+ let queries_buffer = create_query_allocation(&queries_suffix, &context);
+ let keys_buffer = create_attention_cache_allocation(&keys, seq_len, &context);
+ let values_buffer = create_attention_cache_allocation(&values, seq_len, &context);
let total_blocks_count = 32;
let partials_size = num_heads * suffix_length * total_blocks_count * head_dim;
let sums_maxs_size = num_heads * suffix_length * total_blocks_count;
- let mut partials_buffer = context
- .device
- .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let mut sums_buffer = context
- .device
- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let mut maxs_buffer = context
- .device
- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let mut output_buffer = context
- .device
- .new_buffer(
- num_heads * suffix_length * head_dim * std::mem::size_of::<f32>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
- .expect("Failed to create buffer");
+ let mut partials_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), partials_size);
+ let mut sums_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);
+ let mut maxs_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);
+ let mut output_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), num_heads * suffix_length * head_dim);
- // ---- Launch and time ----
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
+ let sinks_allocation: Option<Allocation<Metal>> = None;
kernel_pass1.encode(
&queries_buffer,
&keys_buffer,
@@ -1032,9 +718,9 @@ fn perf_two_pass_attention() {
scale,
num_heads as u32,
suffix_length as u32,
- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,
+ None::<&Allocation<Metal>>,
None,
- sinks_buffer.as_ref().map(|b| b),
+ sinks_allocation.as_ref().map(|b| b),
&mut encoder,
);
kernel_pass2.encode(
@@ -1046,7 +732,6 @@ fn perf_two_pass_attention() {
suffix_length as u32,
&mut encoder,
);
- // Time both host-side and GPU execution
let host_timer = Instant::now();
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
@@ -1062,14 +747,8 @@ fn perf_two_pass_attention() {
host_elapsed_ms
);
- // ---- Sanity check ----
- let output_ptr = output_buffer.contents().as_ptr() as *const f32;
- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * suffix_length * head_dim) };
-
- // Check for NaN/Inf
+ let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);
for &val in output_slice.iter().take(100) {
assert!(val.is_finite(), "Output contains non-finite values");
}
-
- println!("✓ Two-pass attention performance test completed");
}
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
index 1f2746951..a5e4c8073 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs
@@ -1,7 +1,7 @@
use backend_uzu::{
DataType,
backends::common::{
- Backend, DenseBuffer, Encoder, Kernels,
+ Backend, Encoder, Kernels,
gpu_types::{ActivationType, activation_silu_alpha},
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
@@ -13,7 +13,9 @@ use backend_uzu::{
use half::bf16;
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use crate::encodable_block::mlp::moe::tests::common::helpers::{alloc_buffer, alloc_buffer_with_data, create_context};
+use crate::encodable_block::mlp::moe::tests::common::helpers::{
+ alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context,
+};
fn moe_cpu_reference(
x: &[bf16],
@@ -282,40 +284,39 @@ fn run_moe_parity_test_internal<B: Backend>(
let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();
// Create Metal buffers
- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);
- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_weight_bf16);
- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_bias_bf16);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13_gpu);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2_gpu);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);
+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);
+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_weight_bf16);
+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_bias_bf16);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13_gpu);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2_gpu);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
// Allocate intermediate buffers (max capacity)
let max_sumk = t * k;
- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);
- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);
- let mut offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut sumk_buf = alloc_buffer::<B, u32>(&ctx, 1);
+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);
+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);
+ let mut offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut sumk_buf = alloc_allocation::<B, u32>(&ctx, 1);
let num_blocks = ((t + 255) / 256).max(1);
let num_tiles = ((e + 512 - 1) / 512).max(1);
let entries = num_blocks * num_tiles * 512usize;
- let mut partials_buf = alloc_buffer::<B, u32>(&ctx, entries);
- let mut block_bases_buf = alloc_buffer::<B, u32>(&ctx, entries);
- let mut block_alloc_buf = alloc_buffer::<B, u32>(&ctx, entries);
- let mut bucketed_ids_buf = alloc_buffer::<B, i32>(&ctx, max_sumk);
- let mut bucketed_probs_buf = alloc_buffer::<B, bf16>(&ctx, max_sumk);
- let mut tok2row_buf = alloc_buffer::<B, i32>(&ctx, t * k);
- let mut x_perm_buf = alloc_buffer::<B, bf16>(&ctx, max_sumk * d_model);
- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, max_sumk * d_model);
- let mut y_out_buf = alloc_buffer::<B, bf16>(&ctx, t * d_model);
+ let mut partials_buf = alloc_allocation::<B, u32>(&ctx, entries);
+ let mut block_bases_buf = alloc_allocation::<B, u32>(&ctx, entries);
+ let mut block_alloc_buf = alloc_allocation::<B, u32>(&ctx, entries);
+ let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, max_sumk);
+ let mut bucketed_probs_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk);
+ let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);
+ let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk * d_model);
+ let mut y_out_buf = alloc_allocation::<B, bf16>(&ctx, t * d_model);
const BLOCK_M_DECODE: usize = 4; // matches two-pass decode kernel configuration
let h_blocks_decode = (d_ff + BLOCK_M_DECODE - 1) / BLOCK_M_DECODE;
let max_tiles = max_sumk * h_blocks_decode;
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
// Encode ALL kernels in one command buffer
eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");
@@ -386,29 +387,25 @@ fn run_moe_parity_test_internal<B: Backend>(
&mut encoder,
DataType::BF16,
MoeGatherArguments {
- x_buffer: &x_buf,
- bucketed_ids_buffer: &bucketed_ids_buf,
- x_perm_buffer: &mut x_perm_buf,
- sumk_buffer: &sumk_buf,
+ x: &x_buf,
+ bucketed_ids: &bucketed_ids_buf,
+ x_perm: &mut x_perm_buf,
+ sumk: &sumk_buf,
t,
k,
d_model,
},
);
- // Additional buffers for 2-pass
let total_rows = t * k;
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, total_rows * d_ff);
- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, total_rows);
+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, total_rows);
let experts = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("experts");
let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;
let args = MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
+ x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
@@ -431,7 +428,7 @@ fn run_moe_parity_test_internal<B: Backend>(
silu_alpha,
data_type: DataType::BF16,
};
- experts.encode(&mut encoder, args);
+ let y_partial_buf = experts.encode(&mut encoder, args).expect("failed to encode MoE experts");
let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
finalize.encode(
@@ -446,11 +443,11 @@ fn run_moe_parity_test_internal<B: Backend>(
);
eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
eprintln!("[E2E] GPU execution completed");
// Read GPU output
- let y_out_bf16 = unsafe { std::slice::from_raw_parts(y_out_buf.cpu_ptr().as_ptr() as *const bf16, t * d_model) };
+ let y_out_bf16 = allocation_prefix_to_vec::<B, bf16>(&y_out_buf, t * d_model);
let y_out_gpu: Vec<f32> = y_out_bf16.iter().map(|&v| f32::from(v)).collect();
// Validate GPU output is finite
@@ -475,13 +472,10 @@ fn run_moe_parity_test_internal<B: Backend>(
);
// Probe tile bookkeeping
- let total_tiles_cpu =
- unsafe { std::slice::from_raw_parts(total_tiles_buf.cpu_ptr().as_ptr() as *const u32, 2) };
- let dispatch_args_cpu =
- unsafe { std::slice::from_raw_parts(dispatch_args_buf.cpu_ptr().as_ptr() as *const u32, 3) };
- let tile_offsets_cpu =
- unsafe { std::slice::from_raw_parts(tile_offsets_buf.cpu_ptr().as_ptr() as *const u32, (e + 1).min(8)) };
- let sumk_val = unsafe { *(sumk_buf.cpu_ptr().as_ptr() as *const u32) } as usize;
+ let total_tiles_cpu = allocation_prefix_to_vec::<B, u32>(&total_tiles_buf, 2);
+ let dispatch_args_cpu = allocation_prefix_to_vec::<B, u32>(&dispatch_args_buf, 3);
+ let tile_offsets_cpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, (e + 1).min(8));
+ let sumk_val = allocation_to_vec::<B, u32>(&sumk_buf)[0] as usize;
eprintln!("[E2E] Tile bookkeeping:");
eprintln!(
"[E2E] total_tiles={}, dispatch_args=({}, {}, {})",
@@ -492,8 +486,7 @@ fn run_moe_parity_test_internal<B: Backend>(
// For multi-token tests with large d_ff, verify gather output (x_perm)
if t > 1 && d_ff >= 256 {
- let x_perm_cpu =
- unsafe { std::slice::from_raw_parts(x_perm_buf.cpu_ptr().as_ptr() as *const bf16, sumk_val * d_model) };
+ let x_perm_cpu = allocation_prefix_to_vec::<B, bf16>(&x_perm_buf, sumk_val * d_model);
eprintln!("[E2E] x_perm diagnostics (sumk={}):", sumk_val);
eprintln!("[E2E] Row 0 [0:8]: {:?}", &x_perm_cpu[0..8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());
if sumk_val > 1 {
@@ -506,14 +499,11 @@ fn run_moe_parity_test_internal<B: Backend>(
}
// Check tile_map for first few tiles
- let tile_map_cpu = unsafe {
- std::slice::from_raw_parts(tile_map_buf.cpu_ptr().as_ptr() as *const u32, 12.min(max_tiles * 3))
- };
+ let tile_map_cpu = allocation_prefix_to_vec::<B, u32>(&tile_map_buf, 12.min(max_tiles * 3));
eprintln!("[E2E] tile_map (first 4 tiles): {:?}", &tile_map_cpu);
// CRITICAL: Check tok2row mapping for multi-token tests
- let tok2row_cpu =
- unsafe { std::slice::from_raw_parts(tok2row_buf.cpu_ptr().as_ptr() as *const i32, t * k) };
+ let tok2row_cpu = allocation_prefix_to_vec::<B, i32>(&tok2row_buf, t * k);
eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, &tok2row_cpu);
eprintln!(
"[E2E] Expected: token 0→row {}, token 1→row {}",
@@ -526,9 +516,7 @@ fn run_moe_parity_test_internal<B: Backend>(
);
// Check y_partial at specific indices where finalize reads for token 1
- let y_partial_full = unsafe {
- std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sumk_val * d_model)
- };
+ let y_partial_full = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sumk_val * d_model);
eprintln!("[E2E] y_partial spot check:");
eprintln!("[E2E] y_partial[48] (row 0, col 48) = {:.6}", f32::from(y_partial_full[48]));
if sumk_val > 1 {
@@ -612,14 +600,13 @@ fn run_moe_parity_test_internal<B: Backend>(
// Debug: print some sample values from intermediate buffers
eprintln!("[E2E] === DEBUG: Intermediate values ===");
- let topk_ids_gpu = unsafe { std::slice::from_raw_parts(topk_ids_buf.cpu_ptr().as_ptr() as *const i32, t * k) };
- let topk_probs_gpu = unsafe { std::slice::from_raw_parts(topk_probs_buf.cpu_ptr().as_ptr() as *const bf16, t * k) };
+ let topk_ids_gpu = allocation_prefix_to_vec::<B, i32>(&topk_ids_buf, t * k);
+ let topk_probs_gpu = allocation_prefix_to_vec::<B, bf16>(&topk_probs_buf, t * k);
eprintln!("[E2E] TopK IDs: {:?}", topk_ids_gpu);
eprintln!("[E2E] TopK Probs: {:?}", topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>());
- let y_partial_gpu =
- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, max_sumk * d_model) };
- let sumk_actual = unsafe { *(sumk_buf.cpu_ptr().as_ptr() as *const u32) } as usize;
+ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, max_sumk * d_model);
+ let sumk_actual = allocation_to_vec::<B, u32>(&sumk_buf)[0] as usize;
eprintln!("[E2E] sumk={}", sumk_actual);
let sample_size = 16.min(sumk_actual * d_model);
eprintln!(
@@ -682,6 +669,8 @@ fn run_moe_parity_test_internal<B: Backend>(
}
eprintln!("[{}] ✓ PASSED (GPU matches CPU reference)", test_name);
+ drop(y_partial_buf);
+ drop(completed);
}
// Test 1: Minimal (K=1, small dims, no clipping, alpha=1.0)
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
index 41b4a9ae1..41dd6cfec 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs
@@ -9,7 +9,7 @@
use backend_uzu::{
DataType,
backends::common::{
- DenseBuffer, Encoder,
+ Encoder,
gpu_types::{ActivationType, activation_silu_alpha},
kernel::moe::{
MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
@@ -23,7 +23,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::encodable_block::mlp::moe::tests::{
common::{
assert::assert_eq_float,
- helpers::{alloc_buffer, alloc_buffer_with_data, create_context},
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, create_context},
},
cpu_tile_counts, cpu_tile_scan,
};
@@ -378,26 +378,23 @@ fn test_two_pass_decode_correctness() {
);
// Prepare GPU buffers
- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x_perm);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);
- let mut row_expert_map_buf = alloc_buffer_with_data::<B, u32>(&ctx, &row_expert_map);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);
-
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);
- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);
+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
+ let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &row_expert_map);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
// Tile infrastructure
let h_blocks_decode = (d_ff + 3) / 4;
let max_total_tiles = sum_k * h_blocks_decode;
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);
- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_total_tiles * 3);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
// Execute 2-pass decode kernel
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
@@ -406,43 +403,42 @@ fn test_two_pass_decode_correctness() {
const K_TILE: usize = 64;
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;
- experts_kernel.encode(
- &mut encoder,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut y_partial_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k,
- gating_code,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha,
- data_type: DataType::BF16,
- },
- );
-
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let y_partial_buf = experts_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsTwoPassArguments {
+ x_perm: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &mut row_expert_map_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &mut tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ tile_map: &mut tile_map_buf,
+ total_tiles: &mut total_tiles_buf,
+ dispatch_args: &mut dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k,
+ gating_code,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
+
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
// Read GPU partial output and do CPU finalize (weighted sum)
- let y_partial_gpu =
- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sum_k * d_model) };
+ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sum_k * d_model);
// Finalize: y[t] = Σ_k prob[t,k] * y_partial[t*k + k_idx]
let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];
@@ -483,6 +479,8 @@ fn test_two_pass_decode_correctness() {
assert_eq_float(&y_expected, &y_gpu, tolerance, "2-pass decode output");
eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);
+ drop(y_partial_buf);
+ drop(completed);
});
}
@@ -527,65 +525,63 @@ fn test_two_pass_decode_multi_token() {
);
// GPU buffers
- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);
- let mut row_expert_map_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.row_expert_map);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w13);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w2);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);
- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);
-
+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);
+ let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);
// Tile infrastructure
let max_total_tiles = sum_k * ((d_ff + 3) / 4);
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);
- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_total_tiles * 3);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("kernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- experts_kernel.encode(
- &mut encoder,
- MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut y_partial_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &mut tile_counts_buf,
- tile_offsets: &mut tile_offsets_buf,
- tile_map: &mut tile_map_buf,
- total_tiles: &mut total_tiles_buf,
- dispatch_args: &mut dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: ((d_ff + 63) / 64) as u32,
- gating_code,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha,
- data_type: DataType::BF16,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let y_partial_gpu =
- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sum_k * d_model) };
- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
+ let y_partial_buf = experts_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsTwoPassArguments {
+ x_perm: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &mut row_expert_map_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &mut tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ tile_map: &mut tile_map_buf,
+ total_tiles: &mut total_tiles_buf,
+ dispatch_args: &mut dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: ((d_ff + 63) / 64) as u32,
+ gating_code,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sum_k * d_model);
+ let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass decode multi-token");
eprintln!("[2-pass decode multi-token] ✓ PASSED");
+ drop(y_partial_buf);
+ drop(completed);
});
}
@@ -630,32 +626,27 @@ fn test_two_pass_prefill_correctness() {
);
// GPU buffers
- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);
- let mut row_expert_map_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.row_expert_map);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w13);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w2);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);
- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);
-
+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);
+ let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);
// Tile infrastructure
let max_total_tiles = sum_k * ((d_ff + 3) / 4);
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);
- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_total_tiles * 3);
- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);
+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);
+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("kernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
+ x_perm: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &mut row_expert_map_buf,
- hidden_buffer: &mut hidden_buf,
- output_buffer: &mut y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
up_biases: &up_biases_buf,
@@ -678,15 +669,16 @@ fn test_two_pass_prefill_correctness() {
silu_alpha,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut encoder, args);
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let y_partial_buf = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
- let y_partial_gpu =
- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sum_k * d_model) };
- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
+ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sum_k * d_model);
+ let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);
assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass prefill");
eprintln!("[2-pass prefill] ✓ PASSED");
+ drop(y_partial_buf);
+ drop(completed);
});
}
@@ -768,49 +760,46 @@ fn test_fused_single_token_decode() {
);
// GPU buffers
- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);
- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(&ctx, &topk_ids);
- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &topk_probs);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13_all);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2_all);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, k * d_ff);
- let mut y_buf = alloc_buffer::<B, bf16>(&ctx, d_model);
-
+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);
+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(&ctx, &topk_ids);
+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &topk_probs);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13_all);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2_all);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
// Run fused decode kernel
let fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- fused_kernel.encode(
- &mut encoder,
- MoeExpertsSingleDecodeArguments {
- x: &x_buf,
- topk_ids: &topk_ids_buf,
- topk_probs: &topk_probs_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- hidden: &mut hidden_buf,
- y: &mut y_buf,
- d_model,
- d_ff,
- k,
- gating_code,
- silu_alpha: 1.0, // Standard SiLU for testing
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- data_type: DataType::BF16,
- },
- );
-
- encoder.end_encoding().submit().wait_until_completed().unwrap();
+ let y_buf = fused_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code,
+ silu_alpha: 1.0, // Standard SiLU for testing
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
+
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
// Read GPU output
- let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr().as_ptr() as *const bf16, d_model) };
+ let y_gpu = allocation_prefix_to_vec::<B, bf16>(&y_buf, d_model);
// Compute error metrics
let mut max_abs_error = 0.0f32;
@@ -837,9 +826,11 @@ fn test_fused_single_token_decode() {
);
let tolerance = 0.02;
- assert_eq_float(&y_expected, y_gpu, tolerance, "fused single-token output");
+ assert_eq_float(&y_expected, &y_gpu, tolerance, "fused single-token output");
eprintln!("[fused single-token] PASSED (tolerance={:.4})", tolerance);
+ drop(y_buf);
+ drop(completed);
});
}
@@ -895,45 +886,42 @@ fn test_fused_single_token_k4() {
f32::INFINITY,
);
- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);
- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(&ctx, &topk_ids);
- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &topk_probs);
- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13_all);
- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2_all);
- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);
- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);
- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, k * d_ff);
- let mut y_buf = alloc_buffer::<B, bf16>(&ctx, d_model);
-
+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);
+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(&ctx, &topk_ids);
+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &topk_probs);
+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13_all);
+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2_all);
+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);
+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);
let fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(&ctx).expect("fused kernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
- fused_kernel.encode(
- &mut encoder,
- MoeExpertsSingleDecodeArguments {
- x: &x_buf,
- topk_ids: &topk_ids_buf,
- topk_probs: &topk_probs_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- hidden: &mut hidden_buf,
- y: &mut y_buf,
- d_model,
- d_ff,
- k,
- gating_code,
- silu_alpha,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- data_type: DataType::BF16,
- },
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr().as_ptr() as *const bf16, d_model) };
+ let y_buf = fused_kernel
+ .encode(
+ &mut encoder,
+ MoeExpertsSingleDecodeArguments {
+ x: &x_buf,
+ topk_ids: &topk_ids_buf,
+ topk_probs: &topk_probs_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ d_model,
+ d_ff,
+ k,
+ gating_code,
+ silu_alpha,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ data_type: DataType::BF16,
+ },
+ )
+ .expect("failed to encode MoE experts");
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
+
+ let y_gpu = allocation_prefix_to_vec::<B, bf16>(&y_buf, d_model);
// Compute error metrics
let mut max_abs_error = 0.0f32;
@@ -960,8 +948,10 @@ fn test_fused_single_token_k4() {
);
let tolerance = 0.02;
- assert_eq_float(y_gpu, &y_expected, tolerance, "fused single-token K=4 output");
+ assert_eq_float(&y_gpu, &y_expected, tolerance, "fused single-token K=4 output");
eprintln!("[fused single-token K=4] PASSED (tolerance={:.4})", tolerance);
+ drop(y_buf);
+ drop(completed);
});
}
diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs
index fd9190ac2..85b4de671 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs
@@ -1,11 +1,13 @@
use backend_uzu::backends::common::{
- DenseBuffer, Encoder,
+ Encoder,
kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},
};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::encodable_block::mlp::moe::tests::{
- common::helpers::{alloc_buffer, alloc_buffer_with_data, create_context},
+ common::helpers::{
+ alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context,
+ },
cpu_tile_counts, cpu_tile_scan,
};
@@ -31,8 +33,8 @@ fn test_tile_counts_correctness() {
let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
// GPU buffers
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");
@@ -40,18 +42,16 @@ fn test_tile_counts_correctness() {
tile_kernel.encode_counts(
&mut encoder,
MoeTileCountsArguments {
- offsets_buffer: &offsets_buf,
- tile_counts_buffer: &mut tile_counts_buf,
+ offsets: &offsets_buf,
+ tile_counts: &mut tile_counts_buf,
e,
},
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- // Compare
- let tile_counts_gpu =
- unsafe { std::slice::from_raw_parts(tile_counts_buf.cpu_ptr().as_ptr() as *const u32, e) };
+ let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
- assert_eq!(tile_counts_gpu, &tile_counts_cpu[..], "Tile counts mismatch for E={}", e);
+ assert_eq!(tile_counts_gpu, tile_counts_cpu, "Tile counts mismatch for E={}", e);
eprintln!("[TileCountsTest] ✓ PASSED");
}
@@ -76,9 +76,9 @@ fn test_tile_scan_correctness() {
let (tile_offsets_cpu, total_tiles_cpu) = cpu_tile_scan(&tile_counts);
// GPU buffers
- let tile_counts_buf = alloc_buffer_with_data::<B, u32>(&ctx, &tile_counts);
- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);
- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);
+ let tile_counts_buf = alloc_allocation_with_data::<B, u32>(&ctx, &tile_counts);
+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);
+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");
@@ -86,20 +86,18 @@ fn test_tile_scan_correctness() {
tile_kernel.encode_scan(
&mut encoder,
MoeTileScanArguments {
- tile_counts_buffer: &tile_counts_buf,
- tile_offsets_buffer: &mut tile_offsets_buf,
- total_tiles_buffer: &mut total_tiles_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &mut tile_offsets_buf,
+ total_tiles: &mut total_tiles_buf,
e,
},
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- // Compare
- let tile_offsets_gpu =
- unsafe { std::slice::from_raw_parts(tile_offsets_buf.cpu_ptr().as_ptr() as *const u32, e + 1) };
- let total_tiles_gpu = unsafe { *(total_tiles_buf.cpu_ptr().as_ptr() as *const u32) };
+ let tile_offsets_gpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, e + 1);
+ let total_tiles_gpu = allocation_to_vec::<B, u32>(&total_tiles_buf)[0];
- assert_eq!(tile_offsets_gpu, &tile_offsets_cpu[..], "Tile offsets mismatch for E={}", e);
+ assert_eq!(tile_offsets_gpu, tile_offsets_cpu, "Tile offsets mismatch for E={}", e);
assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);
eprintln!("[TileScanTest] ✓ PASSED");
@@ -119,25 +117,24 @@ fn test_tile_edge_cases() {
let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
tile_kernel.encode_counts(
&mut encoder,
MoeTileCountsArguments {
- offsets_buffer: &offsets_buf,
- tile_counts_buffer: &mut tile_counts_buf,
+ offsets: &offsets_buf,
+ tile_counts: &mut tile_counts_buf,
e,
},
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let tile_counts_gpu =
- unsafe { std::slice::from_raw_parts(tile_counts_buf.cpu_ptr().as_ptr() as *const u32, e) };
+ let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
- assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);
+ assert_eq!(tile_counts_gpu, tile_counts_cpu);
assert!(tile_counts_gpu.iter().all(|&c| c == 0));
eprintln!("[TileEdgeCases] ✓ Empty experts PASSED");
}
@@ -149,25 +146,24 @@ fn test_tile_edge_cases() {
let tile_counts_cpu = cpu_tile_counts(&offsets, 16);
- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);
- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);
+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);
+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernels::new");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
tile_kernel.encode_counts(
&mut encoder,
MoeTileCountsArguments {
- offsets_buffer: &offsets_buf,
- tile_counts_buffer: &mut tile_counts_buf,
+ offsets: &offsets_buf,
+ tile_counts: &mut tile_counts_buf,
e,
},
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let tile_counts_gpu =
- unsafe { std::slice::from_raw_parts(tile_counts_buf.cpu_ptr().as_ptr() as *const u32, e) };
+ let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);
- assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);
+ assert_eq!(tile_counts_gpu, tile_counts_cpu);
// With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]
assert_eq!(tile_counts_gpu, &[1, 1, 1, 2]);
eprintln!("[TileEdgeCases] ✓ Small segments PASSED");
diff --git a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
index 723749022..f52078d55 100644
--- a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
+++ b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs
@@ -1,7 +1,5 @@
#![cfg(metal_backend)]
-use bytemuck;
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use rand::seq::SliceRandom;
// for Vec::shuffle
@@ -20,6 +18,11 @@ use crate::{
session::parameter::{SamplingMethod, SamplingProcessingOrder},
};
+#[path = "../../common/mod.rs"]
+mod common;
+
+use common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec};
+
// Constant seed for reproducible test results
const TEST_SAMPLING_SEED: u64 = 42;
@@ -54,7 +57,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
let batch_size = 2;
let vocab_size = 4;
- let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, batch_size, vocab_size, strategy)
+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, strategy)
.expect("Failed to create argmax kernel");
// Create test data: batch_size=2, vocab_size=4
@@ -65,29 +68,16 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
0.1, 0.5, 2.5, 1.0, // batch 1
];
- let mut logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let seeds_buffer = context
- .device
- .new_buffer_with_data(&vec![0; 16], MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut output_buffer = context
- .device
- .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &test_logits);
+ let seeds_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &[0_u64, 0_u64]);
+ let mut output_buffer = alloc_allocation::<Metal, u32>(context.as_ref(), batch_size);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
&mut logits_buffer,
&seeds_buffer,
- 0,
None,
- 0,
&mut output_buffer,
SamplingMethod::Greedy,
batch_size,
@@ -98,8 +88,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
encoder.end_encoding().submit().wait_until_completed().unwrap();
// Check results
- let result_ptr = output_buffer.contents().as_ptr() as *const u32;
- let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };
+ let results: Vec<u32> = allocation_to_vec(&output_buffer);
assert_eq!(results[0], 1, "First batch should select token 1 (highest logit: 3.0)");
assert_eq!(results[1], 2, "Second batch should select token 2 (highest logit: 2.5)");
@@ -123,7 +112,6 @@ fn test_argmax_sampling() {
test_argmax_sampling_with_strategy(ArgmaxStrategy::SinglePass);
}
-#[allow(dead_code)]
fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
use std::time::Instant;
@@ -143,7 +131,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
const VOCAB: usize = 128000; // 128K
// ---- Kernel ----
- let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, BATCH, VOCAB, strategy)
+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, strategy)
.expect("Failed to create Argmax kernel");
// ---- Build random logits ----
@@ -153,21 +141,12 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
*x = rng.random_range(-6.0f32..6.0f32);
}
- let mut logits_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut logits_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
- let seeds_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let seeds_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds);
- let mut output_buf = context
- .device
- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut output_buf = alloc_allocation::<Metal, u32>(context.as_ref(), BATCH);
// ---- Launch once and time ----
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -176,9 +155,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
.encode(
&mut logits_buf,
&seeds_buf,
- 0,
- None,
- 0,
+ None::<&backend_uzu::backends::common::Allocation<Metal>>,
&mut output_buf,
SamplingMethod::Greedy,
BATCH,
@@ -199,9 +176,8 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
);
// Ensure the kernel produced *some* output (sanity).
- let ptr = output_buf.contents().as_ptr() as *const u32;
- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };
- for &tok in sample_ids {
+ let sample_ids: Vec<u32> = allocation_to_vec(&output_buf);
+ for &tok in &sample_ids {
assert!((tok as usize) < VOCAB, "Sampled id out of range");
}
@@ -227,6 +203,18 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
println!("✓ Argmax correctness verified with {:?} strategy", strategy);
}
+#[test]
+#[ignore = "performance-only check"]
+fn perf_argmax_128k_vocab_single_pass() {
+ perf_argmax_128k_vocab_with_strategy(ArgmaxStrategy::SinglePass);
+}
+
+#[test]
+#[ignore = "performance-only check"]
+fn perf_argmax_128k_vocab_two_pass() {
+ perf_argmax_128k_vocab_with_strategy(ArgmaxStrategy::TwoPass);
+}
+
#[test]
fn test_categorical_sampling() {
let context = match <Metal as Backend>::Context::new() {
@@ -240,8 +228,7 @@ fn test_categorical_sampling() {
let batch_size = 2;
let vocab_size = 4;
- let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)
- .expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");
// Create test data with different probability distributions
// First batch: [1.0, 2.0, 1.5, 0.5] -> softmax: [0.134, 0.366, 0.201, 0.082]
@@ -251,10 +238,7 @@ fn test_categorical_sampling() {
0.0, 1.0, 0.0, 0.0, // batch 1
];
- let mut output_buffer = context
- .device
- .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut output_buffer = alloc_allocation::<Metal, u32>(context.as_ref(), batch_size);
// Run sampling multiple times to check distribution
let num_samples = 1000;
@@ -262,16 +246,10 @@ fn test_categorical_sampling() {
for sample_idx in 0..num_samples {
// Create fresh logits buffer since kernel mutates in-place
- let mut logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &test_logits);
let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; batch_size];
- let seeds_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let seeds_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds_vec);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -279,9 +257,7 @@ fn test_categorical_sampling() {
.encode(
&mut logits_buffer,
&seeds_buffer,
- 0,
None,
- 0,
&mut output_buffer,
SamplingMethod::Stochastic {
temperature: None,
@@ -297,8 +273,7 @@ fn test_categorical_sampling() {
.expect("Categorical sampling should succeed");
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let result_ptr = output_buffer.contents().as_ptr() as *const u32;
- let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };
+ let results: Vec<u32> = allocation_to_vec(&output_buffer);
for (batch_idx, &token) in results.iter().enumerate() {
assert!(
@@ -360,8 +335,7 @@ fn test_categorical_sampling_statistical() {
const NUM_SAMPLES: usize = 5000;
const TOLERANCE: f32 = 0.05; // 5% tolerance
- let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");
// Generate random logits
let mut rng = StdRng::seed_from_u64(42);
@@ -389,26 +363,17 @@ fn test_categorical_sampling_statistical() {
}
}
- let mut output_buffer = context
- .device
- .new_buffer(BATCH * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut output_buffer = alloc_allocation::<Metal, u32>(context.as_ref(), BATCH);
let mut counts = vec![0; BATCH * VOCAB];
// Sample many times
for sample_idx in 0..NUM_SAMPLES {
// Create fresh logits buffer since kernel mutates in-place
- let mut logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);
let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; BATCH];
- let seeds_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let seeds_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds_vec);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -416,9 +381,7 @@ fn test_categorical_sampling_statistical() {
.encode(
&mut logits_buffer,
&seeds_buffer,
- 0,
None,
- 0,
&mut output_buffer,
SamplingMethod::Stochastic {
temperature: None,
@@ -434,8 +397,7 @@ fn test_categorical_sampling_statistical() {
.expect("encode");
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let result_ptr = output_buffer.contents().as_ptr() as *const u32;
- let results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH) };
+ let results: Vec<u32> = allocation_to_vec(&output_buffer);
for (batch_idx, &token) in results.iter().enumerate() {
counts[batch_idx * VOCAB + token as usize] += 1;
@@ -485,8 +447,7 @@ fn perf_categorical_128k_vocab() {
const BATCH: usize = 8;
const VOCAB: usize = 128000; // 128K
- let kernel =
- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");
// Build random logits
let mut rng = StdRng::seed_from_u64(123);
@@ -495,21 +456,12 @@ fn perf_categorical_128k_vocab() {
*x = rng.random_range(-6.0f32..6.0f32);
}
- let mut logits_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut logits_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
- let seeds_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let seeds_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds);
- let mut output_buf = context
- .device
- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut output_buf = alloc_allocation::<Metal, u32>(context.as_ref(), BATCH);
// Launch and time
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -518,9 +470,7 @@ fn perf_categorical_128k_vocab() {
.encode(
&mut logits_buf,
&seeds_buf,
- 0,
None,
- 0,
&mut output_buf,
SamplingMethod::Stochastic {
temperature: None,
@@ -546,9 +496,8 @@ fn perf_categorical_128k_vocab() {
);
// Sanity check
- let ptr = output_buf.contents().as_ptr() as *const u32;
- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };
- for &tok in sample_ids {
+ let sample_ids: Vec<u32> = allocation_to_vec(&output_buf);
+ for &tok in &sample_ids {
assert!((tok as usize) < VOCAB, "Sampled id out of range");
}
@@ -576,22 +525,14 @@ fn test_temperature_gpu_cpu_match() {
let logits: Vec<f32> = (0..BATCH * VOCAB).map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01).collect();
- let logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut processed_buffer = context
- .device
- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);
+ let mut processed_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), logits.len());
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };
+ let gpu_results: Vec<f32> = allocation_to_vec(&processed_buffer);
for (idx, (&logit, &processed)) in logits.iter().zip(gpu_results.iter()).enumerate() {
let expected = logit / TEMPERATURE;
@@ -636,22 +577,14 @@ fn test_minp_gpu_cpu_match() {
*x = rng.random_range(-16.0f32..16.0f32);
}
- let logits_buffer = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut processed_buffer = context
- .device
- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);
+ let mut processed_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), logits.len());
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
- let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
+ let all_results: Vec<f32> = allocation_to_vec(&processed_buffer);
for batch_idx in 0..BATCH {
let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];
@@ -682,8 +615,7 @@ fn test_minp_sampling_exact_match(
},
};
- let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)
- .expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");
// Build logits where some tokens have high probability and others have low
// For min_p filtering, tokens with probability < min_p * max_prob are masked
@@ -708,35 +640,24 @@ fn test_minp_sampling_exact_match(
high_prob_token_sets.push(high_prob_tokens);
}
- let mut output_buf = context
- .device
- .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut output_buf = alloc_allocation::<Metal, u32>(context.as_ref(), batch_size);
let num_samples = 1000;
let mut counter = vec![0i32; batch_size * vocab_size];
for draw in 0..num_samples {
// Create fresh logits buffer since kernel mutates in-place
- let mut logits_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let mut logits_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
- let seeds_buf = context
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
+ let seeds_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
&mut logits_buf,
&seeds_buf,
- 0,
None,
- 0,
&mut output_buf,
SamplingMethod::Stochastic {
temperature: None,
@@ -752,8 +673,7 @@ fn test_minp_sampling_exact_match(
.expect("encode");
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let ptr = output_buf.contents().as_ptr() as *const u32;
- let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };
+ let sampled_ids: Vec<u32> = allocation_to_vec(&output_buf);
for (i, &sampled_id) in sampled_ids.iter().enumerate() {
assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");
diff --git a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs
index 3b60230f7..95a66311b 100644
--- a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs
+++ b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs
@@ -1,14 +1,19 @@
#![cfg(metal_backend)]
use crate::{
- ArrayContextExt, DataType,
+ DataType,
backends::{
- common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
+ common::{Allocation, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
metal::Metal,
},
forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
};
+#[path = "../../common/mod.rs"]
+mod common;
+
+use common::helpers::{alloc_allocation_with_data, allocation_to_vec, write_allocation};
+
#[derive(Debug)]
struct Scenario {
name: &'static str,
@@ -40,31 +45,44 @@ fn make_test_layer(
} => window_length + suffix_capacity,
};
let shape = [total_len.max(1), 1, 1];
- let keys = context.create_array_zeros(&shape, DataType::F32, "kv_cache_keys");
- let values = context.create_array_zeros(&shape, DataType::F32, "kv_cache_values");
+
+ let zeroes = vec![0.0_f32; shape.iter().product()];
+ let keys = alloc_allocation_with_data(context, &zeroes);
+ let values = alloc_allocation_with_data(context, &zeroes);
KVCacheLayer {
state,
keys,
values,
+ shape,
+ data_type: DataType::F32,
}
}
+fn overwrite_allocation(
+ allocation: &mut Allocation<Metal>,
+ updates: &[(usize, f32)],
+) {
+ let mut data: Vec<f32> = allocation_to_vec(allocation);
+ for (index, value) in updates {
+ data[*index] = *value;
+ }
+ write_allocation(allocation, &data);
+}
+
fn fill_arrays(layer: &mut KVCacheLayer<Metal>) -> (Vec<f32>, Vec<f32>) {
let initial_keys = {
- let slice = layer.keys.as_slice_mut::<f32>();
- for (idx, value) in slice.iter_mut().enumerate() {
- *value = 1_000.0 + idx as f32;
- }
- slice.to_vec()
+ let len = layer.shape.iter().product();
+ let data: Vec<f32> = (0..len).map(|idx| 1_000.0 + idx as f32).collect();
+ write_allocation(&mut layer.keys, &data);
+ data
};
let initial_values = {
- let slice = layer.values.as_slice_mut::<f32>();
- for (idx, value) in slice.iter_mut().enumerate() {
- *value = 2_000.0 + idx as f32;
- }
- slice.to_vec()
+ let len = layer.shape.iter().product();
+ let data: Vec<f32> = (0..len).map(|idx| 2_000.0 + idx as f32).collect();
+ write_allocation(&mut layer.values, &data);
+ data
};
(initial_keys, initial_values)
@@ -154,10 +172,10 @@ fn run_scenario(
layer.register_accepted_tokens(scenario.number_of_accepted_tokens);
- let actual_keys = layer.keys.as_slice::<f32>().to_vec();
+ let actual_keys: Vec<f32> = allocation_to_vec(&layer.keys);
assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);
- let actual_values = layer.values.as_slice::<f32>().to_vec();
+ let actual_values: Vec<f32> = allocation_to_vec(&layer.values);
assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);
match &layer.state {
@@ -290,17 +308,13 @@ fn kv_cache_slice_apply_contiguous_window() {
let slice = layer.slice(&context, 0..2).expect("slice should exist");
// Mutate the captured slots.
- {
- layer.keys.as_slice_mut::<f32>()[0] = -1.0;
- layer.keys.as_slice_mut::<f32>()[1] = -2.0;
- layer.values.as_slice_mut::<f32>()[0] = -3.0;
- layer.values.as_slice_mut::<f32>()[1] = -4.0;
- }
+ overwrite_allocation(&mut layer.keys, &[(0, -1.0), (1, -2.0)]);
+ overwrite_allocation(&mut layer.values, &[(0, -3.0), (1, -4.0)]);
layer.apply_slice(&slice, None);
- let keys_after = layer.keys.as_slice::<f32>().to_vec();
- let values_after = layer.values.as_slice::<f32>().to_vec();
+ let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);
+ let values_after: Vec<f32> = allocation_to_vec(&layer.values);
assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");
assert_eq!(values_after[0..4], initial_values[0..4], "values restored for contiguous slice");
}
@@ -325,17 +339,13 @@ fn kv_cache_slice_apply_wrap_window() {
let slice = layer.slice(&context, 2..4).expect("slice should exist");
// Captured slots are expected to wrap; mutate them.
- {
- layer.keys.as_slice_mut::<f32>()[2] = -11.0;
- layer.keys.as_slice_mut::<f32>()[3] = -12.0;
- layer.values.as_slice_mut::<f32>()[2] = -13.0;
- layer.values.as_slice_mut::<f32>()[3] = -14.0;
- }
+ overwrite_allocation(&mut layer.keys, &[(2, -11.0), (3, -12.0)]);
+ overwrite_allocation(&mut layer.values, &[(2, -13.0), (3, -14.0)]);
layer.apply_slice(&slice, None);
- let keys_after = layer.keys.as_slice::<f32>().to_vec();
- let values_after = layer.values.as_slice::<f32>().to_vec();
+ let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);
+ let values_after: Vec<f32> = allocation_to_vec(&layer.values);
assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");
assert_eq!(values_after[0..4], initial_values[0..4], "values restored for wrapped slice");
}
diff --git a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs
index ca591a5b0..db5e5ea56 100644
--- a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs
@@ -1,19 +1,22 @@
-use std::{
- fmt::{Debug, Display},
- ops::{Deref, DerefMut},
-};
+use std::fmt::{Debug, Display};
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
- ArrayContextExt, ArrayElement, DataType,
+ ArrayElement, DataType,
backends::{
common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel},
cpu::Cpu,
},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
fused_up: Box<[T]>,
@@ -52,14 +55,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let fused_len = (input.m * 2 * input.h) as usize;
let out_len = (input.m * input.h) as usize;
- let fused_up_array = context.create_array_from(&[fused_len], &input.fused_up, "");
- let hidden_array = context.create_array_uninitialized(&[out_len], T::data_type(), "");
+ let fused_up_allocation = alloc_allocation_with_data::<B, T>(&context, &input.fused_up[..fused_len]);
+ let mut hidden_allocation = alloc_allocation::<B, T>(&context, out_len);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- fused_up_array.buffer().borrow().deref(),
- hidden_array.buffer().borrow_mut().deref_mut(),
- None::<&B::DenseBuffer>,
+ &fused_up_allocation,
+ &mut hidden_allocation,
+ None::<&backend_uzu::backends::common::Allocation<B>>,
input.h,
input.m,
input.act_type,
@@ -67,7 +70,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- hidden_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&hidden_allocation)
}
fn test<T: ArrayElement + Float + Debug + Display>(act_type: ActivationType) {
diff --git a/crates/backend-uzu/tests/unit/kernel/activation_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_test.rs
index 9faed48a9..32466262e 100644
--- a/crates/backend-uzu/tests/unit/kernel/activation_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/activation_test.rs
@@ -1,19 +1,22 @@
-use std::{
- fmt::{Debug, Display},
- ops::{Deref, DerefMut},
-};
+use std::fmt::{Debug, Display};
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
- ArrayContextExt, ArrayElement, DataType,
+ ArrayElement, DataType,
backends::{
common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::ActivationKernel},
cpu::Cpu,
},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
data: Box<[T]>,
@@ -38,30 +41,24 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let n = input.data.len();
if input.in_place {
- let output_array = context.create_array_from(&[n], &input.data, "");
+ let mut output_allocation = alloc_allocation_with_data::<B, T>(&context, &input.data);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
kernel.encode(
- Option::<&B::DenseBuffer>::None,
- output_array.buffer().borrow_mut().deref_mut(),
+ None::<&backend_uzu::backends::common::Allocation<B>>,
+ &mut output_allocation,
n as u32,
input.act_type,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&output_allocation)
} else {
- let input_array = context.create_array_from(&[n], &input.data, "");
- let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");
+ let input_allocation = alloc_allocation_with_data::<B, T>(&context, &input.data);
+ let mut output_allocation = alloc_allocation::<B, T>(&context, n);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
- kernel.encode(
- Some(input_array.buffer().borrow().deref()),
- output_array.buffer().borrow_mut().deref_mut(),
- n as u32,
- input.act_type,
- &mut encoder,
- );
+ kernel.encode(Some(&input_allocation), &mut output_allocation, n as u32, input.act_type, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&output_allocation)
}
}
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs
index 250114a25..b1dff7fde 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs
@@ -1,19 +1,25 @@
-use std::{
- fmt::{Debug, Display},
- ops::{Deref, DerefMut},
-};
+use std::fmt::{Debug, Display};
use backend_uzu::{
- ArrayContextExt, ArrayElement, DataType,
+ ArrayElement, DataType,
backends::{
- common::{Backend, Context, Encoder, kernel::attention::AttentionGemmArguments},
+ common::{
+ Backend, Context, Encoder,
+ kernel::attention::{AttentionGemmArguments, AttentionGemmBlock},
+ },
cpu::Cpu,
},
};
use half::{bf16, f16};
use num_traits::Float;
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
queries: Box<[T]>,
@@ -79,57 +85,46 @@ fn get_test_data<T: ArrayElement + Float>(
fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let block = backend_uzu::backends::common::kernel::attention::AttentionGemmBlock::<B>::new(T::data_type());
+ let block = AttentionGemmBlock::<B>::new(T::data_type());
- let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");
- let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");
- let values_array = context.create_array_from(&[input.values.len()], &input.values, "");
+ let queries_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.queries);
+ let keys_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.keys);
+ let values_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.values);
let output_size = input.suffix_length * input.num_heads * input.head_dim;
- let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");
+ let mut output_allocation = alloc_allocation::<B, T>(context.as_ref(), output_size);
let segment_prefix_length = input.sequence_length - input.suffix_length;
- {
- let q_buf_rc = queries_array.buffer();
- let q_buf = q_buf_rc.borrow();
- let k_buf_rc = keys_array.buffer();
- let k_buf = k_buf_rc.borrow();
- let v_buf_rc = values_array.buffer();
- let v_buf = v_buf_rc.borrow();
- let o_buf_rc = output_array.buffer();
- let mut o_buf = o_buf_rc.borrow_mut();
-
- let args = AttentionGemmArguments::<B> {
- queries_buffer: q_buf.deref(),
- keys_buffer: k_buf.deref(),
- values_buffer: v_buf.deref(),
- output_buffer: o_buf.deref_mut(),
- trie_buffer: None,
- sinks_buffer: None,
- num_heads: input.num_heads,
- num_groups: input.num_kv_heads,
- suffix_length: input.suffix_length,
- sequence_length: input.sequence_length,
- segment_prefix_length,
- max_sequence_length: input.sequence_length,
- ring_params: None,
- head_dim: input.head_dim,
- sliding_window_size: None,
- is_causal: input.do_causal,
- scale: input.scale,
- k_head_stride: input.head_dim as u64,
- k_seq_stride: (input.num_kv_heads * input.head_dim) as u64,
- v_head_stride: input.head_dim as u64,
- v_seq_stride: (input.num_kv_heads * input.head_dim) as u64,
- };
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- block.encode(&context, &mut encoder, args).expect("Failed to encode AttentionGemm");
- encoder.end_encoding().submit().wait_until_completed().unwrap();
- }
+ let args = AttentionGemmArguments::<B> {
+ queries: &queries_allocation,
+ keys: &keys_allocation,
+ values: &values_allocation,
+ output: &mut output_allocation,
+ trie: None,
+ sinks: None,
+ num_heads: input.num_heads,
+ num_groups: input.num_kv_heads,
+ suffix_length: input.suffix_length,
+ sequence_length: input.sequence_length,
+ segment_prefix_length,
+ max_sequence_length: input.sequence_length,
+ ring_params: None,
+ head_dim: input.head_dim,
+ sliding_window_size: None,
+ is_causal: input.do_causal,
+ scale: input.scale,
+ k_head_stride: (input.sequence_length * input.head_dim) as u64,
+ k_seq_stride: input.head_dim as u64,
+ v_head_stride: (input.sequence_length * input.head_dim) as u64,
+ v_seq_stride: input.head_dim as u64,
+ };
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ block.encode(&mut encoder, args).expect("Failed to encode AttentionGemm");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&output_allocation)
}
fn test_internal<T: ArrayElement + Float + Debug + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
index 71ed83fd0..ac8e23339 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
@@ -1,19 +1,22 @@
-use std::{
- fmt::{Debug, Display},
- ops::{Deref, DerefMut},
-};
+use std::fmt::{Debug, Display};
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
- ArrayContextExt, ArrayElement, DataType,
+ ArrayElement, DataType,
backends::{
- common::{Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},
+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},
cpu::Cpu,
},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
queries: Box<[T]>,
@@ -90,19 +93,19 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
)
.expect("Failed to create AttentionSinglePassKernel");
- let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");
- let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");
- let values_array = context.create_array_from(&[input.values.len()], &input.values, "");
+ let queries_allocation = alloc_allocation_with_data::<B, T>(&context, &input.queries);
+ let keys_allocation = alloc_allocation_with_data::<B, T>(&context, &input.keys);
+ let values_allocation = alloc_allocation_with_data::<B, T>(&context, &input.values);
let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;
- let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");
+ let mut output_allocation = alloc_allocation::<B, T>(&context, output_size);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- queries_array.buffer().borrow().deref(),
- keys_array.buffer().borrow().deref(),
- values_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ &queries_allocation,
+ &keys_allocation,
+ &values_allocation,
+ &mut output_allocation,
input.gqa_factor,
input.sequence_length,
input.sequence_length * input.head_dim,
@@ -111,16 +114,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
input.head_dim,
None,
input.scale,
- None::<&B::DenseBuffer>,
+ None::<&Allocation<B>>,
None,
- None::<&B::DenseBuffer>,
+ None::<&Allocation<B>>,
input.num_heads,
input.suffix_length,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&output_allocation)
}
fn test_internal<T: ArrayElement + Float + Debug + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
index 5416ac401..64102c020 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -100,27 +99,33 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassI
)
.expect("Failed to create AttentionTwoPass1Kernel");
- let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");
- let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");
- let values_array = context.create_array_from(&[input.values.len()], &input.values, "");
+ let queries_array = context.create_array_from(&[input.queries.len()], &input.queries);
+ let keys_array = context.create_array_from(&[input.keys.len()], &input.keys);
+ let values_array = context.create_array_from(&[input.values.len()], &input.values);
let total_offsets = (input.suffix_length * input.num_heads) as usize;
let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * input.head_dim as usize;
let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;
- let partials_array = context.create_array_uninitialized(&[partials_size], DataType::F32, "");
- let sums_array = context.create_array_uninitialized(&[sums_size], DataType::F32, "");
- let maxs_array = context.create_array_uninitialized(&[maxs_size], DataType::F32, "");
+ let mut partials = context
+ .create_array_uninitialized(&[partials_size], DataType::F32)
+ .into_allocation();
+ let mut sums = context
+ .create_array_uninitialized(&[sums_size], DataType::F32)
+ .into_allocation();
+ let mut maxs = context
+ .create_array_uninitialized(&[maxs_size], DataType::F32)
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- queries_array.buffer().borrow().deref(),
- keys_array.buffer().borrow().deref(),
- values_array.buffer().borrow().deref(),
- partials_array.buffer().borrow_mut().deref_mut(),
- sums_array.buffer().borrow_mut().deref_mut(),
- maxs_array.buffer().borrow_mut().deref_mut(),
+ queries_array.allocation(),
+ keys_array.allocation(),
+ values_array.allocation(),
+ &mut partials,
+ &mut sums,
+ &mut maxs,
input.gqa_factor,
input.sequence_length,
input.sequence_length * input.head_dim,
@@ -131,17 +136,17 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassI
input.scale,
input.num_heads,
input.suffix_length,
- None::<&B::DenseBuffer>,
+ None::<&backend_uzu::backends::common::Allocation<B>>,
None,
- None::<&B::DenseBuffer>,
+ None::<&backend_uzu::backends::common::Allocation<B>>,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
FirstPassOutput {
- partials: partials_array.as_slice().to_vec(),
- sums: sums_array.as_slice().to_vec(),
- maxs: maxs_array.as_slice().to_vec(),
+ partials: crate::common::helpers::allocation_to_vec(&partials),
+ sums: crate::common::helpers::allocation_to_vec(&sums),
+ maxs: crate::common::helpers::allocation_to_vec(&maxs),
}
}
@@ -199,26 +204,28 @@ fn get_second_pass_output<T: ArrayElement + Float, B: Backend>(input: &SecondPas
<<B as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(&context, T::data_type(), input.head_dim)
.expect("Failed to create AttentionTwoPass2Kernel");
- let partials_array = context.create_array_from(&[input.partials.len()], &input.partials, "");
- let sums_array = context.create_array_from(&[input.sums.len()], &input.sums, "");
- let maxs_array = context.create_array_from(&[input.maxs.len()], &input.maxs, "");
+ let partials_array = context.create_array_from(&[input.partials.len()], &input.partials);
+ let sums_array = context.create_array_from(&[input.sums.len()], &input.sums);
+ let maxs_array = context.create_array_from(&[input.maxs.len()], &input.maxs);
let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;
- let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");
+ let mut output = context
+ .create_array_uninitialized(&[output_size], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- partials_array.buffer().borrow().deref(),
- sums_array.buffer().borrow().deref(),
- maxs_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ partials_array.allocation(),
+ sums_array.allocation(),
+ maxs_array.allocation(),
+ &mut output,
input.num_heads,
input.suffix_length,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
// --- First pass tests ---
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs
index b651c57c4..d7aaed133 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -41,21 +40,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let cache_size = input.num_groups as usize * input.max_sequence_length as usize * input.head_dim as usize;
- let rotated_keys_array = input.rotated_keys.as_ref().map(|rk| context.create_array_from(&[rk.len()], rk, ""));
- let rotated_keys_buf_rc = rotated_keys_array.as_ref().map(|a| a.buffer());
- let rotated_keys_buf_borrow = rotated_keys_buf_rc.as_ref().map(|rc| rc.borrow());
- let rotated_keys_buffer: Option<&B::DenseBuffer> = rotated_keys_buf_borrow.as_ref().map(|b| b.deref());
+ let rotated_keys = input
+ .rotated_keys
+ .as_ref()
+ .map(|rk| context.create_array_from(&[rk.len()], rk).into_allocation());
- let qkv_array = context.create_array_from(&[input.qkv.len()], &input.qkv, "");
- let key_cache_array = context.create_array_from(&[cache_size], &input.key_cache, "");
- let value_cache_array = context.create_array_from(&[cache_size], &input.value_cache, "");
+ let qkv_array = context.create_array_from(&[input.qkv.len()], &input.qkv);
+ let mut key_cache = context
+ .create_array_from(&[cache_size], &input.key_cache)
+ .into_allocation();
+ let mut value_cache = context
+ .create_array_from(&[cache_size], &input.value_cache)
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- rotated_keys_buffer,
- qkv_array.buffer().borrow().deref(),
- key_cache_array.buffer().borrow_mut().deref_mut(),
- value_cache_array.buffer().borrow_mut().deref_mut(),
+ rotated_keys.as_ref(),
+ qkv_array.allocation(),
+ &mut key_cache,
+ &mut value_cache,
input.num_groups,
input.num_heads,
input.head_dim,
@@ -66,7 +69,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (key_cache_array.as_slice().to_vec(), value_cache_array.as_slice().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&key_cache),
+ crate::common::helpers::allocation_to_vec(&value_cache),
+ )
}
fn get_test_data_basic<T: ArrayElement + Float>(keys_in_place: bool) -> Input<T> {
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
index bcd3df4c9..c0c9e50e3 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
@@ -1,4 +1,4 @@
-use std::{fmt::Debug, ops::DerefMut};
+use std::fmt::Debug;
use backend_uzu::{
ArrayContextExt, ArrayElement,
@@ -28,20 +28,22 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidGateKernel::new(&context, T::data_type())
.expect("Failed to create SigmoidGateKernel");
- let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice(), "gate");
- let output_array = context.create_array_from(&[size], &output_data.to_vec().into_boxed_slice(), "output");
+ let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice());
+ let mut output = context
+ .create_array_from(&[size], &output_data.to_vec().into_boxed_slice())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
let total_elements = config.suffix_length * config.num_heads * config.head_dim;
kernel.encode(
- &*gate_array.buffer().borrow(),
- output_array.buffer().borrow_mut().deref_mut(),
+ gate_array.allocation(),
+ &mut output,
total_elements,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {
diff --git a/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rs b/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rs
index 78a863dba..66835a6bc 100644
--- a/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rs
@@ -1,7 +1,7 @@
#![cfg(metal_backend)]
use backend_uzu::{
- ArrayContextExt, DataType,
+ ArrayContextExt, DataType, allocation_to_vec,
backends::{
common::{
Backend, Context, Encoder, Kernels,
@@ -28,15 +28,13 @@ use crate::uzu_test;
macro_rules! borrow_array_buffer {
($name:ident = $array:expr) => {
- let $name = $array.buffer();
- let $name = $name.borrow();
+ let $name = $array.allocation();
};
}
macro_rules! borrow_array_buffer_mut {
($name:ident = $array:expr) => {
- let $name = $array.buffer();
- let mut $name = $name.borrow_mut();
+ let $name = &mut $array;
};
}
@@ -81,19 +79,20 @@ fn audio_conv1d_replicate_matches_reference_f32() {
let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07 - 1.3).cos() * 0.2).collect();
let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.01 - 0.02).collect();
- let mut input = context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");
+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");
+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32);
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_conv1d_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_conv1d_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
- let output = context.create_array_zeros(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");
+ let mut output =
+ context.create_array_zeros(&[batch_size, cout, seq_len_out], DataType::F32).into_allocation();
let expected = conv1d_reference(Conv1dSpec {
input: &input_values,
weight: &weight_values,
@@ -119,11 +118,11 @@ fn audio_conv1d_replicate_matches_reference_f32() {
borrow_array_buffer_mut!(output_buffer = output);
borrow_array_buffer!(lengths_buffer = lengths);
kernel.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input_buffer,
+ weight_buffer,
+ bias_buffer,
+ output_buffer,
+ lengths_buffer,
cin as i32,
cout as i32,
seq_len_in as i32,
@@ -138,7 +137,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {
);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
for index in 0..output_len {
let delta = (expected[index] - got[index]).abs();
assert!(
@@ -174,19 +173,21 @@ fn audio_causal_conv1d_matches_reference_f32() {
let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.02 - 0.5).cos() * 0.1).collect();
let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.001 - 0.002).collect();
- let mut input = context.create_array_zeros(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");
+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len], DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");
+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32);
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv1d_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths_array = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");
+ let mut lengths_array = context.create_array_zeros(&[batch_size], DataType::I32);
lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);
- let output = context.create_array_zeros(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");
+ let mut output = context
+ .create_array_zeros(&[batch_size, cout, seq_len], DataType::F32)
+ .into_allocation();
let expected = causal_conv1d_reference(CausalConv1dSpec {
input: &input_values,
weight: &weight_values,
@@ -208,11 +209,11 @@ fn audio_causal_conv1d_matches_reference_f32() {
borrow_array_buffer_mut!(output_buffer = output);
borrow_array_buffer!(lengths_buffer = lengths_array);
kernel.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input_buffer,
+ weight_buffer,
+ bias_buffer,
+ output_buffer,
+ lengths_buffer,
cin as i32,
cout as i32,
seq_len as i32,
@@ -224,7 +225,7 @@ fn audio_causal_conv1d_matches_reference_f32() {
);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
for index in 0..output_len {
let delta = (expected[index] - got[index]).abs();
assert!(
@@ -264,27 +265,25 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
let bias_values: Vec<f32> = vec![0.01, -0.02];
let mut input =
- context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");
+ context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
let mut weight = context.create_array_zeros(
&[cout, cin / groups, kernel_size],
- DataType::F32,
- "audio_causal_conv_transpose_weight",
- );
+ DataType::F32);
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
- let output = context.create_array_zeros(
+ let mut output = context
+ .create_array_zeros(
&[batch_size, cout, seq_len_out],
- DataType::F32,
- "audio_causal_conv_transpose_output",
- );
+ DataType::F32)
+ .into_allocation();
let expected = causal_conv_transpose1d_reference(CausalConvTranspose1dSpec {
input: &input_values,
weight: &weight_values,
@@ -307,11 +306,11 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
borrow_array_buffer_mut!(output_buffer = output);
borrow_array_buffer!(lengths_buffer = lengths);
kernel.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input_buffer,
+ weight_buffer,
+ bias_buffer,
+ output_buffer,
+ lengths_buffer,
cin as i32,
cout as i32,
seq_len_in as i32,
@@ -323,7 +322,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
for index in 0..output_len {
let delta = (expected[index] - got[index]).abs();
assert!(
@@ -393,28 +392,26 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {
("NSC", 1_i32, [batch_size, seq_len_in, cin], input_values_nsc.as_slice()),
] {
let mut input =
- context.create_array_zeros(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");
+ context.create_array_zeros(&input_shape, DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(input_values);
let mut weight = context.create_array_zeros(
&[cout, cin / groups, kernel_size],
- DataType::F32,
- "audio_causal_conv_transpose_lalamo_weight",
- );
+ DataType::F32);
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
let mut lengths =
- context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");
+ context.create_array_zeros(&[batch_size], DataType::I32);
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
- let output = context.create_array_zeros(
+ let mut output = context
+ .create_array_zeros(
&[batch_size, cout, seq_len_out],
- DataType::F32,
- "audio_causal_conv_transpose_lalamo_output",
- );
+ DataType::F32)
+ .into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
@@ -423,11 +420,11 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {
borrow_array_buffer_mut!(output_buffer = output);
borrow_array_buffer!(lengths_buffer = lengths);
kernel.encode(
- &*input_buffer,
- &*weight_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input_buffer,
+ weight_buffer,
+ bias_buffer,
+ output_buffer,
+ lengths_buffer,
cin as i32,
cout as i32,
seq_len_in as i32,
@@ -441,7 +438,7 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {
);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
for index in 0..output_len {
let delta = (expected[index] - got[index]).abs();
assert!(
@@ -465,20 +462,20 @@ fn audio_leaky_relu_matches_reference_f32() {
let eps = 1e-6f32;
let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
- let mut input = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");
+ let mut input = context.create_array_zeros(&[1, 1, n], DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let alpha = context.create_array_zeros(&[1], DataType::F32, "audio_leaky_relu_alpha");
- let output = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");
+ let alpha = context.create_array_zeros(&[1], DataType::F32);
+ let mut output = context.create_array_zeros(&[1, 1, n], DataType::F32).into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
borrow_array_buffer!(alpha_buffer = alpha);
borrow_array_buffer_mut!(output_buffer = output);
kernel.encode(
- &*input_buffer,
- &*alpha_buffer,
- &mut *output_buffer,
+ input_buffer,
+ alpha_buffer,
+ output_buffer,
1,
n as i32,
0,
@@ -489,7 +486,7 @@ fn audio_leaky_relu_matches_reference_f32() {
);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
for index in 0..n {
let expected = if input_values[index] >= 0.0 {
input_values[index]
@@ -510,18 +507,18 @@ fn audio_tanh_matches_reference_f32() {
let n = 1024usize;
let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
- let mut input = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_input");
+ let mut input = context.create_array_zeros(&[n], DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let output = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_output");
+ let mut output = context.create_array_zeros(&[n], DataType::F32).into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
borrow_array_buffer_mut!(output_buffer = output);
- kernel.encode(Some(&*input_buffer), &mut *output_buffer, n as u32, ActivationType::TANH, command_buffer);
+ kernel.encode(Some(input_buffer), output_buffer, n as u32, ActivationType::TANH, command_buffer);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
for index in 0..n {
let expected = input_values[index].tanh();
let delta = (expected - got[index]).abs();
@@ -539,22 +536,22 @@ fn audio_add_matches_reference_f32() {
let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();
let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();
- let mut a = context.create_array_zeros(&[n], DataType::F32, "audio_add_a");
+ let mut a = context.create_array_zeros(&[n], DataType::F32);
a.as_slice_mut::<f32>().copy_from_slice(&a_values);
- let mut b = context.create_array_zeros(&[n], DataType::F32, "audio_add_b");
+ let mut b = context.create_array_zeros(&[n], DataType::F32);
b.as_slice_mut::<f32>().copy_from_slice(&b_values);
- let sum = context.create_array_zeros(&[n], DataType::F32, "audio_add_sum");
+ let mut sum = context.create_array_zeros(&[n], DataType::F32).into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(a_buffer = a);
borrow_array_buffer!(b_buffer = b);
borrow_array_buffer_mut!(sum_buffer = sum);
- add_kernel.encode(&*a_buffer, &*b_buffer, &mut *sum_buffer, n as i32, command_buffer);
+ add_kernel.encode(a_buffer, b_buffer, sum_buffer, n as i32, command_buffer);
});
- let sum_got = sum.as_slice::<f32>();
+ let sum_got = allocation_to_vec::<Metal, f32>(&sum);
for index in 0..n {
let expected_sum = a_values[index] + b_values[index];
@@ -587,22 +584,24 @@ fn audio_half_snake_matches_reference_f32() {
let alpha_values: Vec<f32> = (0..snake_channels).map(|i| 0.5 + i as f32 * 0.1).collect();
let mut input =
- context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");
+ context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut alpha = context.create_array_zeros(&[snake_channels], DataType::F32, "audio_half_snake_alpha");
+ let mut alpha = context.create_array_zeros(&[snake_channels], DataType::F32);
alpha.as_slice_mut::<f32>().copy_from_slice(&alpha_values);
- let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");
+ let mut output = context
+ .create_array_zeros(&[batch_size, channels, seq_len], DataType::F32)
+ .into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
borrow_array_buffer!(alpha_buffer = alpha);
borrow_array_buffer_mut!(output_buffer = output);
kernel.encode(
- &*input_buffer,
- &*alpha_buffer,
- &mut *output_buffer,
+ input_buffer,
+ alpha_buffer,
+ output_buffer,
channels as i32,
seq_len as i32,
snake_channels as i32,
@@ -613,7 +612,7 @@ fn audio_half_snake_matches_reference_f32() {
);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
let expected = half_snake_reference(HalfSnakeSpec {
input: &input_values,
alpha: &alpha_values,
@@ -692,19 +691,21 @@ fn audio_norm_ncs_matches_reference_f32() {
};
for subtract_mean in [false, true] {
- let mut input = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");
+ let mut input = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut scales = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_scales");
+ let mut scales = context.create_array_zeros(&[channels], DataType::F32);
scales.as_slice_mut::<f32>().copy_from_slice(&scales_values);
- let mut bias = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_bias");
+ let mut bias = context.create_array_zeros(&[channels], DataType::F32);
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_norm_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);
- let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");
+ let mut output = context
+ .create_array_zeros(&[batch_size, channels, seq_len], DataType::F32)
+ .into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
@@ -713,11 +714,11 @@ fn audio_norm_ncs_matches_reference_f32() {
borrow_array_buffer_mut!(output_buffer = output);
borrow_array_buffer!(lengths_buffer = lengths);
kernel.encode(
- &*input_buffer,
- &*scales_buffer,
- &*bias_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ input_buffer,
+ scales_buffer,
+ bias_buffer,
+ output_buffer,
+ lengths_buffer,
channels as i32,
seq_len as i32,
epsilon,
@@ -731,7 +732,7 @@ fn audio_norm_ncs_matches_reference_f32() {
);
});
- let got = output.as_slice::<f32>();
+ let got = allocation_to_vec::<Metal, f32>(&output);
let expected = reference(subtract_mean);
for index in 0..input_len {
let delta = (expected[index] - got[index]).abs();
@@ -758,29 +759,29 @@ fn audio_fsq_decode_matches_reference() {
let num_levels: Vec<i32> = vec![8, 5];
let dim_base_index: Vec<i32> = vec![1, 8];
- let mut tokens = context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");
+ let mut tokens = context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32);
tokens.as_slice_mut::<i32>().copy_from_slice(&[
0, 7, 11, // g0 tokens at t0..t2
3, 5, 9, // g1 tokens at t0..t2
]);
- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);
lengths.as_slice_mut::<i32>().copy_from_slice(&[2]);
- let output = context.create_array_zeros(
+ let mut output = context
+ .create_array_zeros(
&[batch_size, num_groups * codebook_dim, seq_len],
- DataType::F32,
- "audio_fsq_output",
- );
+ DataType::F32)
+ .into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(tokens_buffer = tokens);
borrow_array_buffer_mut!(output_buffer = output);
borrow_array_buffer!(lengths_buffer = lengths);
kernel.encode(
- &*tokens_buffer,
- &mut *output_buffer,
- &*lengths_buffer,
+ tokens_buffer,
+ output_buffer,
+ lengths_buffer,
num_groups as i32,
seq_len as i32,
codebook_dim as i32,
@@ -794,7 +795,7 @@ fn audio_fsq_decode_matches_reference() {
let expected =
fsq_decode_reference(&[0, 7, 11, 3, 5, 9], &[2], batch_size, num_groups, seq_len, codebook_dim, &num_levels)
.expect("reference decode");
- let actual = output.as_slice::<f32>();
+ let actual = allocation_to_vec::<Metal, f32>(&output);
for (index, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() {
let delta = (a - e).abs();
@@ -826,25 +827,24 @@ fn audio_fsq_encode_matches_reference() {
let mut input = context.create_array_zeros(
&[batch_size, num_groups * codebook_dim, seq_len],
- DataType::F32,
- "audio_fsq_encode_input",
- );
+ DataType::F32);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);
- let tokens =
- context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");
+ let mut tokens = context
+ .create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32)
+ .into_allocation();
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
borrow_array_buffer_mut!(tokens_buffer = tokens);
borrow_array_buffer!(lengths_buffer = lengths);
kernel.encode(
- &*input_buffer,
- &mut *tokens_buffer,
- &*lengths_buffer,
+ input_buffer,
+ tokens_buffer,
+ lengths_buffer,
num_groups as i32,
seq_len as i32,
codebook_dim as i32,
@@ -868,7 +868,7 @@ fn audio_fsq_encode_matches_reference() {
eps,
)
.expect("reference encode");
- let actual = tokens.as_slice::<i32>();
+ let actual = allocation_to_vec::<Metal, i32>(&tokens);
assert_eq!(actual, expected);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs
index 88d3007ca..78b32d9e2 100644
--- a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs
@@ -1,7 +1,5 @@
#![cfg(metal_backend)]
-use std::ops::{Deref, DerefMut};
-
use backend_uzu::{
ArrayContextExt, DataType,
backends::{
@@ -30,20 +28,20 @@ fn run_conv_update<B: Backend>(
) -> (Vec<f32>, Vec<f32>) {
let context = B::Context::new().expect("Failed to create context");
- let w_array = context.create_array_from(&[w.len()], w, "w");
- let b_array = context.create_array_from(&[b.len()], b, "b");
- let in_out_array = context.create_array_from(&[in_proj.len()], in_proj, "in_out");
- let state_array = context.create_array_from(&[state.len()], state, "state");
+ let w_array = context.create_array_from(&[w.len()], w);
+ let b_array = context.create_array_from(&[b.len()], b);
+ let mut in_out = context.create_array_from(&[in_proj.len()], in_proj).into_allocation();
+ let mut state_allocation = context.create_array_from(&[state.len()], state).into_allocation();
let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(&context, DataType::F32, true)
.expect("Failed to create kernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- w_array.buffer().borrow().deref(),
- Some(b_array.buffer().borrow().deref()),
- in_out_array.buffer().borrow_mut().deref_mut(),
- state_array.buffer().borrow_mut().deref_mut(),
+ w_array.allocation(),
+ Some(b_array.allocation()),
+ &mut in_out,
+ &mut state_allocation,
kernel_size,
conv_dim,
state_stride,
@@ -51,8 +49,8 @@ fn run_conv_update<B: Backend>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let out = in_out_array.as_slice::<f32>()[..conv_dim as usize].to_vec();
- let new_state = state_array.as_slice::<f32>().to_vec();
+ let out = crate::common::helpers::allocation_prefix_to_vec::<B, f32>(&in_out, conv_dim as usize);
+ let new_state = crate::common::helpers::allocation_to_vec::<B, f32>(&state_allocation);
(out, new_state)
}
@@ -71,24 +69,24 @@ fn run_delta_net_update<B: Backend>(
) -> (Vec<f32>, Vec<f32>) {
let context = B::Context::new().expect("Failed to create context");
- let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");
- let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");
- let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");
- let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");
- let state_array = context.create_array_from(&[state.len()], state, "state");
- let out_array = context.create_array_zeros(&[value_dim as usize], DataType::F32, "out");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj);
+ let a_log_array = context.create_array_from(&[a_log.len()], a_log);
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias);
+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight);
+ let mut state_allocation = context.create_array_from(&[state.len()], state).into_allocation();
+ let mut out = context.create_array_zeros(&[value_dim as usize], DataType::F32).into_allocation();
let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetUpdateKernel::new(&context, DataType::F32, head_k_dim)
.expect("Failed to create kernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- in_proj_array.buffer().borrow().deref(),
- a_log_array.buffer().borrow().deref(),
- dt_bias_array.buffer().borrow().deref(),
- norm_weight_array.buffer().borrow().deref(),
- state_array.buffer().borrow_mut().deref_mut(),
- out_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.allocation(),
+ a_log_array.allocation(),
+ dt_bias_array.allocation(),
+ norm_weight_array.allocation(),
+ &mut state_allocation,
+ &mut out,
num_v_heads,
num_k_heads,
head_v_dim,
@@ -99,8 +97,8 @@ fn run_delta_net_update<B: Backend>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let out = out_array.as_slice::<f32>().to_vec();
- let new_state = state_array.as_slice::<f32>().to_vec();
+ let out = crate::common::helpers::allocation_to_vec::<B, f32>(&out);
+ let new_state = crate::common::helpers::allocation_to_vec::<B, f32>(&state_allocation);
(out, new_state)
}
@@ -176,14 +174,15 @@ fn test_delta_net_conv_scan() {
// Test: Conv1dPack + DeltaNetConvScan on Metal
let context = <Metal as Backend>::Context::new().expect("context");
- let state_array = context.create_array_from(&[init_state.len()], &init_state, "state");
- let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
- let w_array = context.create_array_from(&[w.len()], &w, "w");
- let b_array = context.create_array_from(&[b.len()], &b, "b");
+ let state_array = context.create_array_from(&[init_state.len()], &init_state);
+ let mut in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj).into_allocation();
+ let w_array = context.create_array_from(&[w.len()], &w);
+ let b_array = context.create_array_from(&[b.len()], &b);
let padded_len = (tap_count + suffix_len) * total_proj_dim;
- let padded_array = context.create_array_zeros(&[padded_len], DataType::F32, "padded");
- let state_out_array = context.create_array_zeros(&[conv_dim * tap_count], DataType::F32, "state_out");
+ let mut padded_array = context.create_array_zeros(&[padded_len], DataType::F32).into_allocation();
+ let mut state_out_array =
+ context.create_array_zeros(&[conv_dim * tap_count], DataType::F32).into_allocation();
let pack_kernel =
<<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32).expect("pack");
@@ -193,9 +192,9 @@ fn test_delta_net_conv_scan() {
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
pack_kernel.encode(
- state_array.buffer().borrow().deref(),
- in_proj_array.buffer().borrow().deref(),
- padded_array.buffer().borrow_mut().deref_mut(),
+ state_array.allocation(),
+ &in_proj_array,
+ &mut padded_array,
tap_count as u32,
total_proj_dim as u32,
suffix_len as u32,
@@ -203,11 +202,11 @@ fn test_delta_net_conv_scan() {
&mut encoder,
);
scan_kernel.encode(
- padded_array.buffer().borrow().deref(),
- w_array.buffer().borrow().deref(),
- Some(b_array.buffer().borrow().deref()),
- in_proj_array.buffer().borrow_mut().deref_mut(),
- state_out_array.buffer().borrow_mut().deref_mut(),
+ &padded_array,
+ w_array.allocation(),
+ Some(b_array.allocation()),
+ &mut in_proj_array,
+ &mut state_out_array,
suffix_len as u32,
kernel_size as u32,
total_proj_dim as u32,
@@ -218,13 +217,13 @@ fn test_delta_net_conv_scan() {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let in_proj_result = in_proj_array.as_slice::<f32>();
+ let in_proj_result: Vec<f32> = crate::common::helpers::allocation_to_vec(&in_proj_array);
let mut scan_outputs = vec![0.0f32; suffix_len * conv_dim];
for t in 0..suffix_len {
scan_outputs[t * conv_dim..(t + 1) * conv_dim]
.copy_from_slice(&in_proj_result[t * total_proj_dim..t * total_proj_dim + conv_dim]);
}
- let scan_state = state_out_array.as_slice::<f32>().to_vec();
+ let scan_state: Vec<f32> = crate::common::helpers::allocation_to_vec(&state_out_array);
assert_close(&ref_outputs, &scan_outputs, 1e-4, 1e-3, "ConvScan output");
assert_close(&ref_state, &scan_state, 1e-5, 1e-4, "ConvScan state");
@@ -308,17 +307,21 @@ fn run_prefill_with_norm_gate(
let num_dv_groups = (head_v_dim as u32 + 7) / 8;
let context = <Metal as Backend>::Context::new().expect("context");
- let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");
- let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");
- let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");
- let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");
- let state_array = context.create_array_from(&[state.len()], state, "state");
- let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");
- let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");
- let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");
-
- let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
- let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj);
+ let a_log_array = context.create_array_from(&[a_log.len()], a_log);
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias);
+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight);
+ let mut state_array = context.create_array_from(&[state.len()], state).into_allocation();
+ let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32).into_allocation();
+ let mut q_norm_array =
+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+ let mut k_norm_array =
+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+
+ let mut beta_array =
+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
+ let mut decay_array =
+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&context,
@@ -337,13 +340,13 @@ fn run_prefill_with_norm_gate(
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
prep_k.encode(
- in_proj_array.buffer().borrow().deref(),
- a_log_array.buffer().borrow().deref(),
- dt_bias_array.buffer().borrow().deref(),
- q_norm_array.buffer().borrow_mut().deref_mut(),
- k_norm_array.buffer().borrow_mut().deref_mut(),
- beta_array.buffer().borrow_mut().deref_mut(),
- decay_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.allocation(),
+ a_log_array.allocation(),
+ dt_bias_array.allocation(),
+ &mut q_norm_array,
+ &mut k_norm_array,
+ &mut beta_array,
+ &mut decay_array,
num_v_heads as u32,
num_k_heads as u32,
key_dim as u32,
@@ -352,13 +355,13 @@ fn run_prefill_with_norm_gate(
&mut encoder,
);
prefill_k.encode(
- q_norm_array.buffer().borrow().deref(),
- k_norm_array.buffer().borrow().deref(),
- beta_array.buffer().borrow().deref(),
- decay_array.buffer().borrow().deref(),
- in_proj_array.buffer().borrow().deref(),
- state_array.buffer().borrow_mut().deref_mut(),
- out_array.buffer().borrow_mut().deref_mut(),
+ &q_norm_array,
+ &k_norm_array,
+ &beta_array,
+ &decay_array,
+ in_proj_array.allocation(),
+ &mut state_array,
+ &mut out_array,
num_v_heads as u32,
num_k_heads as u32,
head_v_dim as u32,
@@ -369,9 +372,9 @@ fn run_prefill_with_norm_gate(
&mut encoder,
);
norm_k.encode(
- out_array.buffer().borrow_mut().deref_mut(),
- in_proj_array.buffer().borrow().deref(),
- norm_weight_array.buffer().borrow().deref(),
+ &mut out_array,
+ in_proj_array.allocation(),
+ norm_weight_array.allocation(),
num_v_heads as u32,
head_v_dim as u32,
value_dim as u32,
@@ -383,7 +386,10 @@ fn run_prefill_with_norm_gate(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (out_array.as_slice::<f32>().to_vec(), state_array.as_slice::<f32>().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&out_array),
+ crate::common::helpers::allocation_to_vec(&state_array),
+ )
}
fn test_prefill_norm_gate_impl(
@@ -469,13 +475,15 @@ fn test_delta_net_prefill_prep() {
// CPU reference via Kernels trait
let cpu_ctx = <Cpu as Backend>::Context::new().expect("cpu context");
- let cpu_in_proj = cpu_ctx.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
- let cpu_a_log = cpu_ctx.create_array_from(&[a_log.len()], &a_log, "a_log");
- let cpu_dt_bias = cpu_ctx.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");
- let cpu_q = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q");
- let cpu_k = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k");
- let cpu_beta = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
- let cpu_decay = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+ let cpu_in_proj = cpu_ctx.create_array_from(&[in_proj.len()], &in_proj);
+ let cpu_a_log = cpu_ctx.create_array_from(&[a_log.len()], &a_log);
+ let cpu_dt_bias = cpu_ctx.create_array_from(&[dt_bias.len()], &dt_bias);
+ let mut cpu_q = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+ let mut cpu_k = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+ let mut cpu_beta =
+ cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
+ let mut cpu_decay =
+ cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
let cpu_prep = <<Cpu as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&cpu_ctx,
@@ -485,13 +493,13 @@ fn test_delta_net_prefill_prep() {
.unwrap();
let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");
cpu_prep.encode(
- cpu_in_proj.buffer().borrow().deref(),
- cpu_a_log.buffer().borrow().deref(),
- cpu_dt_bias.buffer().borrow().deref(),
- cpu_q.buffer().borrow_mut().deref_mut(),
- cpu_k.buffer().borrow_mut().deref_mut(),
- cpu_beta.buffer().borrow_mut().deref_mut(),
- cpu_decay.buffer().borrow_mut().deref_mut(),
+ cpu_in_proj.allocation(),
+ cpu_a_log.allocation(),
+ cpu_dt_bias.allocation(),
+ &mut cpu_q,
+ &mut cpu_k,
+ &mut cpu_beta,
+ &mut cpu_decay,
num_v_heads as u32,
num_k_heads as u32,
key_dim as u32,
@@ -501,21 +509,25 @@ fn test_delta_net_prefill_prep() {
);
cpu_enc.end_encoding().submit().wait_until_completed().unwrap();
- let ref_q = cpu_q.as_slice::<f32>().to_vec();
- let ref_k = cpu_k.as_slice::<f32>().to_vec();
- let ref_beta = cpu_beta.as_slice::<f32>().to_vec();
- let ref_decay = cpu_decay.as_slice::<f32>().to_vec();
+ let ref_q: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_q);
+ let ref_k: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_k);
+ let ref_beta: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_beta);
+ let ref_decay: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_decay);
// Metal
let context = <Metal as Backend>::Context::new().expect("context");
- let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
- let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");
- let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");
- let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");
- let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");
-
- let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
- let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj);
+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log);
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias);
+ let mut q_norm_array =
+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+ let mut k_norm_array =
+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+
+ let mut beta_array =
+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
+ let mut decay_array =
+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(
&context,
@@ -526,13 +538,13 @@ fn test_delta_net_prefill_prep() {
let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
prep_k.encode(
- in_proj_array.buffer().borrow().deref(),
- a_log_array.buffer().borrow().deref(),
- dt_bias_array.buffer().borrow().deref(),
- q_norm_array.buffer().borrow_mut().deref_mut(),
- k_norm_array.buffer().borrow_mut().deref_mut(),
- beta_array.buffer().borrow_mut().deref_mut(),
- decay_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.allocation(),
+ a_log_array.allocation(),
+ dt_bias_array.allocation(),
+ &mut q_norm_array,
+ &mut k_norm_array,
+ &mut beta_array,
+ &mut decay_array,
num_v_heads as u32,
num_k_heads as u32,
key_dim as u32,
@@ -542,15 +554,15 @@ fn test_delta_net_prefill_prep() {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let gpu_q = q_norm_array.as_slice::<f32>();
- let gpu_k = k_norm_array.as_slice::<f32>();
- let gpu_beta = beta_array.as_slice::<f32>();
- let gpu_decay = decay_array.as_slice::<f32>();
+ let gpu_q: Vec<f32> = crate::common::helpers::allocation_to_vec(&q_norm_array);
+ let gpu_k: Vec<f32> = crate::common::helpers::allocation_to_vec(&k_norm_array);
+ let gpu_beta: Vec<f32> = crate::common::helpers::allocation_to_vec(&beta_array);
+ let gpu_decay: Vec<f32> = crate::common::helpers::allocation_to_vec(&decay_array);
- assert_close(gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");
- assert_close(gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");
- assert_close(gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");
- assert_close(gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");
+ assert_close(&gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");
+ assert_close(&gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");
+ assert_close(&gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");
+ assert_close(&gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");
}
#[uzu_test]
@@ -576,16 +588,20 @@ fn bench_delta_net_prefill() {
let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();
let context = <Metal as Backend>::Context::new().expect("context");
- let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");
- let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");
- let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");
- let norm_weight_array = context.create_array_from(&[norm_weight.len()], &norm_weight, "norm_weight");
- let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");
- let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");
- let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");
-
- let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");
- let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");
+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj);
+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log);
+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias);
+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], &norm_weight);
+ let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32).into_allocation();
+ let mut q_norm_array =
+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+ let mut k_norm_array =
+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();
+
+ let mut beta_array =
+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
+ let mut decay_array =
+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();
let num_dv_groups = (head_v_dim as u32 + 7) / 8;
@@ -613,13 +629,13 @@ fn bench_delta_net_prefill() {
let prep_result = run_perf_with_warmup("prep_only", 5, 50, || {
let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
prep_k.encode(
- in_proj_array.buffer().borrow().deref(),
- a_log_array.buffer().borrow().deref(),
- dt_bias_array.buffer().borrow().deref(),
- q_norm_array.buffer().borrow_mut().deref_mut(),
- k_norm_array.buffer().borrow_mut().deref_mut(),
- beta_array.buffer().borrow_mut().deref_mut(),
- decay_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.allocation(),
+ a_log_array.allocation(),
+ dt_bias_array.allocation(),
+ &mut q_norm_array,
+ &mut k_norm_array,
+ &mut beta_array,
+ &mut decay_array,
num_v_heads as u32,
num_k_heads as u32,
key_dim as u32,
@@ -632,20 +648,19 @@ fn bench_delta_net_prefill() {
prep_result.print();
// Benchmark prep + prefill + norm_gate (production path)
- let mut state_array = context.create_array_zeros(&[state_size], DataType::F32, "state");
+ let mut state_array = context.create_array_zeros(&[state_size], DataType::F32).into_allocation();
let prefill_result = run_perf_with_warmup("prep+prefill+norm_gate", 5, 50, || {
- state_array.as_slice_mut::<f32>().fill(0.0);
-
let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
+ encoder.encode_fill(&mut state_array, 0);
prep_k.encode(
- in_proj_array.buffer().borrow().deref(),
- a_log_array.buffer().borrow().deref(),
- dt_bias_array.buffer().borrow().deref(),
- q_norm_array.buffer().borrow_mut().deref_mut(),
- k_norm_array.buffer().borrow_mut().deref_mut(),
- beta_array.buffer().borrow_mut().deref_mut(),
- decay_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.allocation(),
+ a_log_array.allocation(),
+ dt_bias_array.allocation(),
+ &mut q_norm_array,
+ &mut k_norm_array,
+ &mut beta_array,
+ &mut decay_array,
num_v_heads as u32,
num_k_heads as u32,
key_dim as u32,
@@ -654,13 +669,13 @@ fn bench_delta_net_prefill() {
&mut encoder,
);
prefill_k.encode(
- q_norm_array.buffer().borrow().deref(),
- k_norm_array.buffer().borrow().deref(),
- beta_array.buffer().borrow().deref(),
- decay_array.buffer().borrow().deref(),
- in_proj_array.buffer().borrow().deref(),
- state_array.buffer().borrow_mut().deref_mut(),
- out_array.buffer().borrow_mut().deref_mut(),
+ &q_norm_array,
+ &k_norm_array,
+ &beta_array,
+ &decay_array,
+ in_proj_array.allocation(),
+ &mut state_array,
+ &mut out_array,
num_v_heads as u32,
num_k_heads as u32,
head_v_dim as u32,
@@ -671,9 +686,9 @@ fn bench_delta_net_prefill() {
&mut encoder,
);
norm_k.encode(
- out_array.buffer().borrow_mut().deref_mut(),
- in_proj_array.buffer().borrow().deref(),
- norm_weight_array.buffer().borrow().deref(),
+ &mut out_array,
+ in_proj_array.allocation(),
+ norm_weight_array.allocation(),
num_v_heads as u32,
head_v_dim as u32,
value_dim as u32,
diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs
index 8d67f4c70..89342d224 100644
--- a/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -105,16 +104,18 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let kernel = <<B as Backend>::Kernels as Kernels>::EmbeddingRowsSumKernel::new(&context, T::data_type())
.expect("Failed to create EmbeddingRowsSumKernel");
- let token_indices_array = context.create_array_from(&[input.num_rows as usize], &input.token_indices, "");
+ let token_indices_array = context.create_array_from(&[input.num_rows as usize], &input.token_indices);
let weights_array =
- context.create_array_from(&[input.total_rows as usize, input.model_dim as usize], &input.weights, "");
- let output_array = context.create_array_uninitialized(&[input.model_dim as usize], T::data_type(), "");
+ context.create_array_from(&[input.total_rows as usize, input.model_dim as usize], &input.weights);
+ let mut output = context
+ .create_array_uninitialized(&[input.model_dim as usize], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- token_indices_array.buffer().borrow().deref(),
- weights_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ token_indices_array.allocation(),
+ weights_array.allocation(),
+ &mut output,
input.num_rows,
input.total_rows,
input.model_dim,
@@ -123,7 +124,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_basic<T: ArrayElement + Float + Debug + Display>() {
diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs
index 0ac84188a..23012072c 100644
--- a/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::Debug,
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -57,15 +56,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
<<B as Backend>::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(&context, T::data_type())
.expect("Failed to create FullPrecisionEmbeddingLookupKernel");
- let token_ids_array = context.create_array_from(&[input.batch_size], &input.token_ids, "");
- let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights, "");
- let output_array = context.create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type(), "");
+ let token_ids_array = context.create_array_from(&[input.batch_size], &input.token_ids);
+ let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights);
+ let mut output = context
+ .create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
kernel.encode(
- token_ids_array.buffer().borrow().deref(),
- weights_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ token_ids_array.allocation(),
+ weights_array.allocation(),
+ &mut output,
input.batch_size as u32,
input.vocab_size as u32,
input.model_dim as u32,
@@ -74,7 +75,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test<T: ArrayElement + Float + Debug>() {
diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
index ffeaf4649..56ccc1434 100644
--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -120,20 +119,21 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let weights_stride = input.model_dim as usize / packing_divisor;
let num_groups = (input.model_dim as usize + input.group_size as usize - 1) / input.group_size as usize;
- let token_ids_array = context.create_array_from(&[input.batch_size as usize], &input.token_ids, "");
- let weights_array = context.create_array_from(&[input.vocab_size as usize, weights_stride], &input.weights, "");
- let scales_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.scales, "");
- let biases_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.biases, "");
- let output_array =
- context.create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type(), "");
+ let token_ids_array = context.create_array_from(&[input.batch_size as usize], &input.token_ids);
+ let weights_array = context.create_array_from(&[input.vocab_size as usize, weights_stride], &input.weights);
+ let scales_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.scales);
+ let biases_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.biases);
+ let mut output = context
+ .create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- token_ids_array.buffer().borrow().deref(),
- weights_array.buffer().borrow().deref(),
- scales_array.buffer().borrow().deref(),
- biases_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ token_ids_array.allocation(),
+ weights_array.allocation(),
+ scales_array.allocation(),
+ biases_array.allocation(),
+ &mut output,
input.batch_size,
input.vocab_size,
input.model_dim,
@@ -142,7 +142,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_quant_mode<T: ArrayElement + Float + Debug + Display>(quant_mode: QuantizationMode) {
diff --git a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
index a37515b97..c67f7353b 100644
--- a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::Debug,
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -105,20 +104,22 @@ fn run_kernel<T: ArrayElement + Float, B: Backend>(input: &TestInput<T>) -> Vec<
.expect("Failed to create HadamardTransformKernel");
let total_elements = input.batch_count * input.channel_count;
- let data_array = context.create_array_from(&[total_elements], &input.data, "data");
- let factors_array = context.create_array_from::<i32>(&[input.channel_count], &input.factors, "factors");
+ let mut data = context
+ .create_array_from(&[total_elements], &input.data)
+ .into_allocation();
+ let factors_array = context.create_array_from::<i32>(&[input.channel_count], &input.factors);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- data_array.buffer().borrow_mut().deref_mut(),
- factors_array.buffer().borrow().deref(),
+ &mut data,
+ factors_array.allocation(),
input.channel_count as u32,
input.batch_count as u32,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- data_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&data)
}
fn test_hadamard_transform<T: ArrayElement + Float + Debug>(tolerance: f64) {
diff --git a/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rs b/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rs
index bb8a23ad9..f0c644f01 100644
--- a/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::DerefMut,
};
use backend_uzu::{
@@ -45,13 +44,13 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
.expect("Failed to create KVCacheUpdateKernel");
let total = input.num_heads as usize * input.max_sequence_length as usize * input.head_dim as usize;
- let keys_array = context.create_array_from(&[total], &input.keys, "");
- let values_array = context.create_array_from(&[total], &input.values, "");
+ let mut keys = context.create_array_from(&[total], &input.keys).into_allocation();
+ let mut values = context.create_array_from(&[total], &input.values).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
kernel.encode(
- keys_array.buffer().borrow_mut().deref_mut(),
- values_array.buffer().borrow_mut().deref_mut(),
+ &mut keys,
+ &mut values,
&input.swaps,
input.swaps.len() as u32,
input.num_heads,
@@ -60,7 +59,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (keys_array.as_slice().to_vec(), values_array.as_slice().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&keys),
+ crate::common::helpers::allocation_to_vec(&values),
+ )
}
/// Single swap between two different positions.
diff --git a/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rs
index a6675dca2..d19549361 100644
--- a/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -128,23 +127,21 @@ fn get_output<
.expect("Failed to create LayerNormKernel");
let input_size = input.input.len();
- let input_array = context.create_array_from(&[input_size], &input.input, "");
- let input_array_buffer_rc = input_array.buffer();
- let input_array_borrow = input_array_buffer_rc.borrow();
- let input_array_deref = input_array_borrow.deref();
- let input_buffer = (!input.in_place).then(|| input_array_deref);
-
- let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");
- let output_array = match input.in_place {
- true => context.create_array_from(&[input_size], &input.output, ""),
- false => context.create_array_uninitialized(&[input_size], OUT::data_type(), ""),
+ let input_buffer = (!input.in_place).then(|| context.create_array_from(&[input_size], &input.input).into_allocation());
+
+ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales);
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[input_size], &input.output).into_allocation(),
+ false => context
+ .create_array_uninitialized(&[input_size], OUT::data_type())
+ .into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_buffer,
- scales_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ input_buffer.as_ref(),
+ scales_array.allocation(),
+ &mut output,
input.batch_size,
input.model_dim,
input.epsilon,
@@ -154,7 +151,7 @@ fn get_output<
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_internal<
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
index 19ec918c3..018aa3458 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
@@ -1,15 +1,16 @@
#![cfg(metal_backend)]
use std::{
- ops::{Deref, DerefMut},
rc::Rc,
};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::bf16;
use backend_uzu::{
ArrayContextExt, ArrayElement,
backends::{
common::{
- Backend, Context, Encoder,
+ AllocationType, Backend, Context, Encoder,
kernel::{
ManualKernels,
matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
@@ -19,11 +20,13 @@ use backend_uzu::{
metal::{DeviceExt, MatmulDispatchPath, Metal, MetalContext},
},
};
-use criterion::{BenchmarkId, Criterion, Throughput};
-use half::bf16;
use crate::{
- common::{assert::assert_eq_float, type_short_name},
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation_with_data, allocation_to_vec},
+ type_short_name,
+ },
uzu_bench, uzu_test,
};
@@ -66,21 +69,19 @@ fn get_test_data(
fn get_cpu_output(input: &Input) -> Vec<bf16> {
let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");
- let left_array = cpu_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");
- let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");
- let destination_array = if let Some(ref prefill) = input.destination_prefill {
- cpu_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")
+ let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);
+ let left_allocation = alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &input.left);
+ let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {
+ alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, prefill)
} else {
- cpu_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")
+ cpu_context
+ .create_allocation(
+ input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),
+ AllocationType::Global,
+ )
+ .expect("Failed to create allocation")
};
- let left_buffer = left_array.buffer();
- let left_ref = left_buffer.borrow();
- let right_buffer = right_array.buffer();
- let right_ref = right_buffer.borrow();
- let destination_buffer = destination_array.buffer();
- let mut destination_ref = destination_buffer.borrow_mut();
-
let argument_c = if input.destination_prefill.is_some() {
MatmulArgumentC::Accumulate
} else {
@@ -93,14 +94,13 @@ fn get_cpu_output(input: &Input) -> Vec<bf16> {
let mut encoder = Encoder::new(cpu_context.as_ref()).expect("encoder");
matmul_kernel.encode(
- &cpu_context,
MatmulArguments {
- a: left_ref.deref(),
+ a: &left_allocation,
a_offset: 0,
- b: right_ref.deref(),
+ b: right_array.allocation(),
ab_scale: input.ab_scale,
c: argument_c,
- d: destination_ref.deref_mut(),
+ d: &mut destination_allocation,
batch_dim: input.batch_dim as u32,
input_dim: input.input_dim as u32,
output_dim: input.output_dim as u32,
@@ -108,9 +108,7 @@ fn get_cpu_output(input: &Input) -> Vec<bf16> {
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- drop(destination_ref);
- destination_array.as_slice().to_vec()
+ allocation_to_vec(&destination_allocation)
}
fn get_mpp_output(
@@ -118,21 +116,19 @@ fn get_mpp_output(
matmul_kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,
input: &Input,
) -> Vec<bf16> {
- let left_array = metal_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");
- let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");
- let destination_array = if let Some(ref prefill) = input.destination_prefill {
- metal_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")
+ let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);
+ let left_allocation = alloc_allocation_with_data::<Metal, bf16>(metal_context, &input.left);
+ let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {
+ alloc_allocation_with_data::<Metal, bf16>(metal_context, prefill)
} else {
- metal_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")
+ metal_context
+ .create_allocation(
+ input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),
+ AllocationType::Global,
+ )
+ .expect("Failed to create allocation")
};
- let left_buffer = left_array.buffer();
- let left_ref = left_buffer.borrow();
- let right_buffer = right_array.buffer();
- let right_ref = right_buffer.borrow();
- let destination_buffer = destination_array.buffer();
- let mut destination_ref = destination_buffer.borrow_mut();
-
let argument_c = if input.destination_prefill.is_some() {
MatmulArgumentC::Accumulate
} else {
@@ -141,14 +137,13 @@ fn get_mpp_output(
let mut encoder = Encoder::new(metal_context).expect("encoder");
matmul_kernel.encode_with_path(
- metal_context,
MatmulArguments {
- a: left_ref.deref(),
+ a: &left_allocation,
a_offset: 0,
- b: right_ref.deref(),
+ b: right_array.allocation(),
ab_scale: input.ab_scale,
c: argument_c,
- d: destination_ref.deref_mut(),
+ d: &mut destination_allocation,
batch_dim: input.batch_dim as u32,
input_dim: input.input_dim as u32,
output_dim: input.output_dim as u32,
@@ -157,9 +152,7 @@ fn get_mpp_output(
MatmulDispatchPath::GemmMpp,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- drop(destination_ref);
- destination_array.as_slice().to_vec()
+ allocation_to_vec(&destination_allocation)
}
fn create_metal_context_and_matmul_kernel()
@@ -291,10 +284,13 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {
criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()));
for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {
- let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");
- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");
- let destination_array =
- metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");
+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());
+ let left_allocation = metal_context
+ .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("Failed to create allocation");
+ let mut destination_allocation = metal_context
+ .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("Failed to create allocation");
let floating_point_operations = 2 * batch_dim * input_dim * output_dim;
benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));
@@ -302,26 +298,18 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {
benchmark_group.bench_function(
BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),
|bencher| {
- let left_buffer = left_array.buffer();
- let left_ref = left_buffer.borrow();
- let right_buffer = right_array.buffer();
- let right_ref = right_buffer.borrow();
- let destination_buffer = destination_array.buffer();
-
bencher.iter_custom(|iteration_count| {
let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();
for _ in 0..iteration_count {
- let mut destination_ref = destination_buffer.borrow_mut();
matmul_kernel.encode_with_path(
- &metal_context,
MatmulArguments {
- a: left_ref.deref(),
+ a: &left_allocation,
a_offset: 0,
- b: right_ref.deref(),
+ b: right_array.allocation(),
ab_scale: 1.0,
c: MatmulArgumentC::None,
- d: destination_ref.deref_mut(),
+ d: &mut destination_allocation,
batch_dim: batch_dim as u32,
input_dim: input_dim as u32,
output_dim: output_dim as u32,
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
index 2f197a59f..e1fa1aa86 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
@@ -1,13 +1,12 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
ArrayContextExt, ArrayElement,
backends::{
common::{
- Backend, Context, Encoder,
+ AllocationType, Backend, Context, Encoder,
kernel::{
ManualKernels,
matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
@@ -25,7 +24,13 @@ use num_traits::Float;
#[cfg(metal_backend)]
use crate::{common::type_short_name, uzu_bench};
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation_with_data, allocation_to_vec},
+ },
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
a: Box<[T]>,
@@ -55,40 +60,31 @@ fn get_test_data<T: ArrayElement + Float>(
(input, expected)
}
-fn get_output<T: ArrayElement + Float, B: Backend>(
- input: &Input<T>,
- ab_scale: f32,
-) -> Vec<T> {
+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: f32) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
let m = input.m as u32;
let k = input.k as u32;
let n = input.n as u32;
- let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");
- let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");
- let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");
-
- let a_buf = a_array.buffer();
- let a_ref = a_buf.borrow();
- let b_buf = b_array.buffer();
- let b_ref = b_buf.borrow();
- let d_buf = d_array.buffer();
- let mut d_ref = d_buf.borrow_mut();
+ let b_array = context.create_array_from(&[input.n, input.k], &input.b);
+ let a_allocation = alloc_allocation_with_data::<B, T>(&context, &input.a);
+ let mut d_allocation = context
+ .create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global)
+ .expect("Failed to create allocation");
let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
.expect("Failed to create MatmulKernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- &context,
MatmulArguments {
- a: a_ref.deref(),
+ a: &a_allocation,
a_offset: 0,
- b: b_ref.deref(),
+ b: b_array.allocation(),
ab_scale,
c: MatmulArgumentC::None,
- d: d_ref.deref_mut(),
+ d: &mut d_allocation,
batch_dim: m,
input_dim: k,
output_dim: n,
@@ -96,9 +92,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- drop(d_ref);
- d_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&d_allocation)
}
fn test<T: ArrayElement + Float + Debug + Display>(
@@ -125,12 +119,7 @@ fn test_with_scale<T: ArrayElement + Float + Debug + Display>(
let expected = get_output::<T, Cpu>(&input, ab_scale);
for_each_non_cpu_backend!(|B| {
let output = get_output::<T, B>(&input, ab_scale);
- assert_eq_float(
- &expected,
- &output,
- eps,
- &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()),
- );
+ assert_eq_float(&expected, &output, eps, &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()));
});
}
@@ -210,10 +199,13 @@ fn bench_gemm(criterion: &mut Criterion) {
criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));
for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {
- let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");
- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");
- let destination_array =
- metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");
+ let left_allocation = metal_context
+ .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("Failed to create allocation");
+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());
+ let mut destination_allocation = metal_context
+ .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("Failed to create allocation");
let floating_point_operations = 2 * batch_dim * input_dim * output_dim;
benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));
@@ -221,26 +213,18 @@ fn bench_gemm(criterion: &mut Criterion) {
benchmark_group.bench_function(
BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),
|bencher| {
- let left_buffer = left_array.buffer();
- let left_ref = left_buffer.borrow();
- let right_buffer = right_array.buffer();
- let right_ref = right_buffer.borrow();
- let destination_buffer = destination_array.buffer();
-
bencher.iter_custom(|iteration_count| {
let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();
for _ in 0..iteration_count {
- let mut destination_ref = destination_buffer.borrow_mut();
matmul_kernel.encode_with_path(
- &metal_context,
MatmulArguments {
- a: left_ref.deref(),
+ a: &left_allocation,
a_offset: 0,
- b: right_ref.deref(),
+ b: right_array.allocation(),
ab_scale: 1.0,
c: MatmulArgumentC::None,
- d: destination_ref.deref_mut(),
+ d: &mut destination_allocation,
batch_dim: batch_dim as u32,
input_dim: input_dim as u32,
output_dim: output_dim as u32,
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
index 8ffaa9b97..351338060 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs
@@ -1,13 +1,14 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
ArrayContextExt, ArrayElement,
backends::{
common::{
- Backend, Context, Encoder,
+ AllocationType, Backend, Context, Encoder,
kernel::{
ManualKernels,
matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
@@ -16,10 +17,11 @@ use backend_uzu::{
cpu::Cpu,
},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{assert::assert_eq_float, helpers::{alloc_allocation_with_data, allocation_to_vec}},
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
a: Box<[T]>,
@@ -56,30 +58,24 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let k = input.k as u32;
let n = input.n as u32;
- let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");
- let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");
- let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");
-
- let a_buf = a_array.buffer();
- let a_ref = a_buf.borrow();
- let b_buf = b_array.buffer();
- let b_ref = b_buf.borrow();
- let d_buf = d_array.buffer();
- let mut d_ref = d_buf.borrow_mut();
+ let b_array = context.create_array_from(&[input.n, input.k], &input.b);
+ let a_allocation = alloc_allocation_with_data::<B, T>(&context, &input.a);
+ let mut d_allocation = context
+ .create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global)
+ .expect("Failed to create allocation");
let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type())
.expect("Failed to create MatmulKernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- &context,
MatmulArguments {
- a: a_ref.deref(),
+ a: &a_allocation,
a_offset: 0,
- b: b_ref.deref(),
+ b: b_array.allocation(),
ab_scale: 1.0,
c: MatmulArgumentC::None,
- d: d_ref.deref_mut(),
+ d: &mut d_allocation,
batch_dim: m,
input_dim: k,
output_dim: n,
@@ -87,9 +83,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- drop(d_ref);
- d_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&d_allocation)
}
fn test<T: ArrayElement + Float + Debug + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs
index f650b2f70..3413c8151 100644
--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs
@@ -1,5 +1,4 @@
-use std::ops::{Deref, DerefMut};
-
+use rand::{RngExt, SeedableRng, rngs::StdRng};
use backend_uzu::{
ArrayContextExt, ArrayElement,
backends::{
@@ -7,7 +6,6 @@ use backend_uzu::{
cpu::Cpu,
},
};
-use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::uzu_test;
@@ -40,21 +38,22 @@ fn get_output<B: Backend>(
let topk_ids_len = (t * k).max(1);
let topk_ids_array = if topk_ids.is_empty() {
- context.create_array_uninitialized(&[topk_ids_len], i32::data_type(), "topk_ids")
+ context.create_array_uninitialized(&[topk_ids_len], i32::data_type())
} else {
- context.create_array_from(&[topk_ids_len], topk_ids, "topk_ids")
+ context.create_array_from(&[topk_ids_len], topk_ids)
};
- let offsets_array = context.create_array_uninitialized(&[e + 1], u32::data_type(), "offsets");
- let sum_k_array = context.create_array_uninitialized(&[1], u32::data_type(), "sum_k");
+ let mut offsets = context.create_array_uninitialized(&[e + 1], u32::data_type()).into_allocation();
+ let mut sum_k = context.create_array_uninitialized(&[1], u32::data_type()).into_allocation();
let num_tiles = e.div_ceil(512).max(1);
- let partials_array = context.create_array_uninitialized(&[num_tiles * 512], u32::data_type(), "partials");
+ let mut partials =
+ context.create_array_uninitialized(&[num_tiles * 512], u32::data_type()).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- topk_ids_array.buffer().borrow().deref(),
- offsets_array.buffer().borrow_mut().deref_mut(),
- sum_k_array.buffer().borrow_mut().deref_mut(),
- partials_array.buffer().borrow_mut().deref_mut(),
+ topk_ids_array.allocation(),
+ &mut offsets,
+ &mut sum_k,
+ &mut partials,
t as u32,
e as u32,
k as u32,
@@ -62,9 +61,9 @@ fn get_output<B: Backend>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let offsets = offsets_array.as_slice::<u32>().to_vec();
- let sum_k = sum_k_array.as_slice::<u32>()[0];
- let partials = partials_array.as_slice::<u32>()[..e].to_vec();
+ let offsets = crate::common::helpers::allocation_to_vec::<B, u32>(&offsets);
+ let sum_k = crate::common::helpers::allocation_to_vec::<B, u32>(&sum_k)[0];
+ let partials = crate::common::helpers::allocation_prefix_to_vec::<B, u32>(&partials, e);
(offsets, sum_k, partials)
}
diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rs
index 60fc1b460..e7afacf71 100644
--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rs
@@ -1,5 +1,6 @@
-use std::ops::{Deref, DerefMut};
-
+use half::bf16;
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::StdRng};
use backend_uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
@@ -7,9 +8,6 @@ use backend_uzu::{
cpu::Cpu,
},
};
-use half::bf16;
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::{
common::{assert::assert_eq_float, helpers::create_context},
@@ -27,18 +25,18 @@ struct Input<T: ArrayElement + Float> {
fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = create_context::<B>();
- let tok2row_array = context.create_array_from(&[input.tok2row.len()], &input.tok2row, "");
- let probs_array = context.create_array_from(&[input.probs.len()], &input.probs, "");
- let y_partial_array = context.create_array_from(&[input.y_partial.len()], &input.y_partial, "");
- let y_out_array = context.create_array_uninitialized(&[input.t * input.d_model], T::data_type(), "");
+ let tok2row_array = context.create_array_from(&[input.tok2row.len()], &input.tok2row);
+ let probs_array = context.create_array_from(&[input.probs.len()], &input.probs);
+ let y_partial_array = context.create_array_from(&[input.y_partial.len()], &input.y_partial);
+ let mut y_out = context.create_array_uninitialized(&[input.t * input.d_model], T::data_type()).into_allocation();
let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(&context, DataType::BF16).expect("finalize kernel");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
finalize.encode(
- tok2row_array.buffer().borrow().deref(),
- probs_array.buffer().borrow().deref(),
- y_partial_array.buffer().borrow().deref(),
- y_out_array.buffer().borrow_mut().deref_mut(),
+ tok2row_array.allocation(),
+ probs_array.allocation(),
+ y_partial_array.allocation(),
+ &mut y_out,
input.t as u32,
input.d_model as u32,
input.k as u32,
@@ -46,7 +44,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- y_out_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&y_out)
}
fn test_finalize_internal(
diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs
index 7a7572e16..493348dc4 100644
--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs
@@ -1,7 +1,8 @@
-use std::ops::{Deref, DerefMut};
-
+use half::bf16;
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::StdRng};
use backend_uzu::{
- ArrayContextExt, ArrayElement,
+ ArrayElement,
backends::{
common::{
Backend, Encoder,
@@ -10,12 +11,12 @@ use backend_uzu::{
cpu::Cpu,
},
};
-use half::bf16;
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::{
- common::{assert::assert_eq_float, helpers::create_context},
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},
+ },
uzu_test,
};
@@ -30,11 +31,11 @@ struct Input<T: ArrayElement + Float> {
fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = create_context::<B>();
- let x_array = context.create_array_from(&[input.x.len()], &input.x, "");
- let ids_array = context.create_array_from(&[input.bucket_ids.len()], &input.bucket_ids, "");
let sumk_data: [u32; 1] = [input.sum_k as u32];
- let sumk_array = context.create_array_from(&[1], &sumk_data, "");
- let x_perm_array = context.create_array_uninitialized(&[input.sum_k * input.d_model], T::data_type(), "");
+ let x_allocation = alloc_allocation_with_data::<B, T>(&context, &input.x);
+ let ids_allocation = alloc_allocation_with_data::<B, i32>(&context, &input.bucket_ids);
+ let sumk_allocation = alloc_allocation_with_data::<B, u32>(&context, &sumk_data);
+ let mut x_perm_allocation = alloc_allocation::<B, T>(&context, input.sum_k * input.d_model);
let gather = MoeGatherKernels::<B>::new(&context).expect("MoeGatherKernel::new");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -42,10 +43,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
&mut encoder,
T::data_type(),
MoeGatherArguments {
- x_buffer: &x_array.buffer().borrow().deref(),
- bucketed_ids_buffer: &ids_array.buffer().borrow().deref(),
- x_perm_buffer: x_perm_array.buffer().borrow_mut().deref_mut(),
- sumk_buffer: &sumk_array.buffer().borrow().deref(),
+ x: &x_allocation,
+ bucketed_ids: &ids_allocation,
+ x_perm: &mut x_perm_allocation,
+ sumk: &sumk_allocation,
t: input.t,
k: input.sum_k / input.t, // Decompose sum_k into k per token
d_model: input.d_model,
@@ -53,7 +54,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- x_perm_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&x_perm_allocation)
}
fn test_gather_internal(
diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rs
index 607256bbf..669f465ad 100644
--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::Debug,
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -28,20 +27,20 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
) -> (Vec<i32>, Vec<T>) {
let ctx = create_context::<B>();
- let input_array = ctx.create_array_from(&[input.len()], input, "");
- let weights_array = ctx.create_array_from(&[weights.len()], weights, "");
- let bias_array = ctx.create_array_from(&[bias.len()], bias, "");
- let ids_array = ctx.create_array_uninitialized(&[t * k], DataType::I32, "");
- let probs_array = ctx.create_array_uninitialized(&[t * k], T::data_type(), "");
+ let input_array = ctx.create_array_from(&[input.len()], input);
+ let weights_array = ctx.create_array_from(&[weights.len()], weights);
+ let bias_array = ctx.create_array_from(&[bias.len()], bias);
+ let mut ids = ctx.create_array_uninitialized(&[t * k], DataType::I32).into_allocation();
+ let mut probs = ctx.create_array_uninitialized(&[t * k], T::data_type()).into_allocation();
let kernel = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, T::data_type()).expect("kernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_array.buffer().borrow().deref(),
- weights_array.buffer().borrow().deref(),
- bias_array.buffer().borrow().deref(),
- ids_array.buffer().borrow_mut().deref_mut(),
- probs_array.buffer().borrow_mut().deref_mut(),
+ input_array.allocation(),
+ weights_array.allocation(),
+ bias_array.allocation(),
+ &mut ids,
+ &mut probs,
t as u32,
d_model as u32,
e as u32,
@@ -51,7 +50,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (ids_array.as_slice().to_vec(), probs_array.as_slice().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&ids),
+ crate::common::helpers::allocation_to_vec(&probs),
+ )
}
fn run_router_topk_once<B: Backend, T: ArrayElement + Debug + Float>(
diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rs
index 8ef8cc7a1..7cd64b7b3 100644
--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rs
@@ -1,10 +1,11 @@
-use std::ops::{Deref, DerefMut};
-
+use half::bf16;
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::StdRng};
use backend_uzu::{
- Array, ArrayContextExt, ArrayElement, DataType,
+ ArrayContextExt, ArrayElement, DataType,
backends::{
common::{
- Backend, Encoder, Kernels,
+ Allocation, Backend, Encoder, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,
MoeScatterBucketsKernel,
@@ -13,9 +14,6 @@ use backend_uzu::{
cpu::Cpu,
},
};
-use half::bf16;
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::StdRng};
use crate::{common::helpers::create_context, uzu_test};
@@ -67,22 +65,22 @@ fn get_output_topk<B: Backend, T: ArrayElement + Float>(
d_model: usize,
e: usize,
k: usize,
-) -> (Array<B>, Array<B>) {
- let input_array = ctx.create_array_from(&[input.len()], input, "");
- let weights_array = ctx.create_array_from(&[weights.len()], weights, "");
- let bias_array = ctx.create_array_from(&[bias.len()], bias, "");
- let topk_ids_array = ctx.create_array_uninitialized(&[t * k], DataType::I32, "");
- let topk_probs_array = ctx.create_array_uninitialized(&[t * k], T::data_type(), "");
+) -> (Allocation<B>, Allocation<B>) {
+ let input_array = ctx.create_array_from(&[input.len()], input);
+ let weights_array = ctx.create_array_from(&[weights.len()], weights);
+ let bias_array = ctx.create_array_from(&[bias.len()], bias);
+ let mut topk_ids = ctx.create_array_uninitialized(&[t * k], DataType::I32).into_allocation();
+ let mut topk_probs = ctx.create_array_uninitialized(&[t * k], T::data_type()).into_allocation();
let topk_kernel =
<<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, T::data_type()).expect("router_topk");
let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
topk_kernel.encode(
- input_array.buffer().borrow().deref(),
- weights_array.buffer().borrow().deref(),
- bias_array.buffer().borrow().deref(),
- topk_ids_array.buffer().borrow_mut().deref_mut(),
- topk_probs_array.buffer().borrow_mut().deref_mut(),
+ input_array.allocation(),
+ weights_array.allocation(),
+ bias_array.allocation(),
+ &mut topk_ids,
+ &mut topk_probs,
t as u32,
d_model as u32,
e as u32,
@@ -92,7 +90,7 @@ fn get_output_topk<B: Backend, T: ArrayElement + Float>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (topk_ids_array, topk_probs_array)
+ (topk_ids, topk_probs)
}
fn get_output<B: Backend, T: ArrayElement + Float>(
@@ -103,26 +101,26 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
d_model: usize,
e: usize,
k: usize,
-) -> (Vec<i32>, Vec<T>, Array<B>) {
+) -> (Vec<i32>, Vec<T>, Vec<u32>) {
let ctx = create_context::<B>();
// top-k
- let (topk_ids_array, topk_probs_array) =
+ let (topk_ids, topk_probs) =
get_output_topk::<B, T>(ctx.as_ref(), input, weights, bias, t, d_model, e, k);
// fused
- let offsets_array = ctx.create_array_uninitialized(&[e + 1], DataType::U32, "");
- let sumk_array = ctx.create_array_uninitialized(&[1], DataType::U32, "");
+ let mut offsets = ctx.create_array_uninitialized(&[e + 1], DataType::U32).into_allocation();
+ let mut sumk = ctx.create_array_uninitialized(&[1], DataType::U32).into_allocation();
let num_tiles = e.div_ceil(512).max(1);
- let partials_array = ctx.create_array_uninitialized(&[num_tiles * 512], DataType::U32, "");
+ let mut partials = ctx.create_array_uninitialized(&[num_tiles * 512], DataType::U32).into_allocation();
let fused_kernel =
<<B as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
fused_kernel.encode(
- topk_ids_array.buffer().borrow().deref(),
- offsets_array.buffer().borrow_mut().deref_mut(),
- sumk_array.buffer().borrow_mut().deref_mut(),
- partials_array.buffer().borrow_mut().deref_mut(),
+ &topk_ids,
+ &mut offsets,
+ &mut sumk,
+ &mut partials,
t as u32,
e as u32,
k as u32,
@@ -134,15 +132,15 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
// Partials already created by fused kernel above
let num_blocks = 1; // Fused kernel uses single block
let entries = num_blocks * num_tiles * 512usize;
- let block_bases_array = ctx.create_array_uninitialized(&[entries], DataType::U32, "");
- let block_alloc_array = ctx.create_array_uninitialized(&[entries], DataType::U32, "");
+ let mut block_bases = ctx.create_array_uninitialized(&[entries], DataType::U32).into_allocation();
+ let mut block_alloc = ctx.create_array_uninitialized(&[entries], DataType::U32).into_allocation();
let scatter_bases_kernel = <<B as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(ctx.as_ref())
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
scatter_bases_kernel.encode(
- partials_array.buffer().borrow().deref(),
- block_bases_array.buffer().borrow_mut().deref_mut(),
- block_alloc_array.buffer().borrow_mut().deref_mut(),
+ &partials,
+ &mut block_bases,
+ &mut block_alloc,
e as u32,
num_blocks as u32,
num_tiles as u32,
@@ -150,22 +148,22 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let sumk = unsafe { std::slice::from_raw_parts(sumk_array.cpu_ptr().as_ptr() as *const u32, 1) }[0] as usize;
+ let sumk = crate::common::helpers::allocation_to_vec::<B, u32>(&sumk)[0] as usize;
// scatter
let scatter_kernel = <<B as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, T::data_type())
.expect("Failed to create <<B as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");
- let out_ids_array = ctx.create_array_uninitialized(&[sumk], DataType::I32, "");
- let out_probs_array = ctx.create_array_uninitialized(&[sumk], T::data_type(), "");
+ let mut out_ids = ctx.create_array_uninitialized(&[sumk], DataType::I32).into_allocation();
+ let mut out_probs = ctx.create_array_uninitialized(&[sumk], T::data_type()).into_allocation();
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
scatter_kernel.encode(
- topk_ids_array.buffer().borrow().deref(),
- topk_probs_array.buffer().borrow().deref(),
- offsets_array.buffer().borrow().deref(),
- block_bases_array.buffer().borrow().deref(),
- block_alloc_array.buffer().borrow().deref(),
- out_ids_array.buffer().borrow_mut().deref_mut(),
- out_probs_array.buffer().borrow_mut().deref_mut(),
+ &topk_ids,
+ &topk_probs,
+ &offsets,
+ &block_bases,
+ &block_alloc,
+ &mut out_ids,
+ &mut out_probs,
t as u32,
e as u32,
k as u32,
@@ -175,7 +173,11 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (out_ids_array.as_slice().to_vec(), out_probs_array.as_slice().to_vec(), offsets_array)
+ (
+ crate::common::helpers::allocation_to_vec(&out_ids),
+ crate::common::helpers::allocation_to_vec(&out_probs),
+ crate::common::helpers::allocation_to_vec(&offsets),
+ )
}
fn test_scatter_internal<B: Backend, T: ArrayElement + Float>(
@@ -207,10 +209,10 @@ fn test_scatter_internal<B: Backend, T: ArrayElement + Float>(
e,
k,
);
- let (cpu_ids, _cpu_probs, offsets_cpu) =
- cpu_expert_buckets(topk_ids_cpu.as_slice(), &topk_probs_cpu.as_slice::<T>(), t, e, k);
- let offsets_gpu = unsafe { std::slice::from_raw_parts(out_offsets_array.cpu_ptr().as_ptr() as *const u32, e + 1) };
- assert_eq!(offsets_gpu, &offsets_cpu[..]);
+ let cpu_topk_ids: Vec<i32> = crate::common::helpers::allocation_to_vec(&topk_ids_cpu);
+ let cpu_topk_probs: Vec<T> = crate::common::helpers::allocation_to_vec(&topk_probs_cpu);
+ let (cpu_ids, _cpu_probs, offsets_cpu) = cpu_expert_buckets(&cpu_topk_ids, &cpu_topk_probs, t, e, k);
+ assert_eq!(out_offsets_array, offsets_cpu);
// Compare per-expert multisets of ids
for ei in 0..e {
diff --git a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rs b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rs
index 249482753..052f29639 100644
--- a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -50,13 +49,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;
let output_len = (input.batch_size * input.hidden_dim) as usize;
- let input_array = context.create_array_from(&[input_len], &input.input, "");
- let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");
+ let input_array = context.create_array_from(&[input_len], &input.input);
+ let mut output = context
+ .create_array_uninitialized(&[output_len], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ input_array.allocation(),
+ &mut output,
input.seq_len,
input.hidden_dim,
input.batch_size,
@@ -64,7 +65,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_internal<T: ArrayElement + Float + Debug + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rs b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rs
index b6c1b8104..bb93185a0 100644
--- a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -50,13 +49,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;
let output_len = (input.batch_size * input.hidden_dim) as usize;
- let input_array = context.create_array_from(&[input_len], &input.input, "");
- let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");
+ let input_array = context.create_array_from(&[input_len], &input.input);
+ let mut output = context
+ .create_array_uninitialized(&[output_len], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ input_array.allocation(),
+ &mut output,
input.seq_len,
input.hidden_dim,
input.batch_size,
@@ -64,7 +65,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_internal<T: ArrayElement + Float + Debug + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rs
index 7c5c3c437..ca8b3526d 100644
--- a/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -126,14 +125,14 @@ fn get_output<
.expect("Failed to create QKNormKernel");
let qkv_len = input.qkv.len();
- let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");
- let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");
+ let mut qkv = context.create_array_from(&[qkv_len], &input.qkv).into_allocation();
+ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- None::<&B::DenseBuffer>,
- scales_array.buffer().borrow().deref(),
- qkv_array.buffer().borrow_mut().deref_mut(),
+ None::<&backend_uzu::backends::common::Allocation<B>>,
+ scales_array.allocation(),
+ &mut qkv,
input.batch_size,
input.num_q_heads,
input.num_kv_heads,
@@ -147,7 +146,7 @@ fn get_output<
);
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
- qkv_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&qkv)
}
fn test_internal<
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
index 0168c0ae7..471d15b50 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
@@ -3,7 +3,7 @@ use std::fmt::{Debug, Display};
use backend_uzu::{
ArrayElement, DataType,
backends::{
- common::{Backend, Context, DenseBuffer, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedKernel},
+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedKernel},
cpu::Cpu,
},
};
@@ -15,7 +15,10 @@ use rand::{RngExt, SeedableRng, rngs::SmallRng};
use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
use crate::{
- common::{helpers::alloc_buffer_with_data, type_short_name},
+ common::{
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ type_short_name,
+ },
uzu_bench, uzu_test,
};
@@ -191,16 +194,15 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
)
.expect("Failed to create QuantizedMatmulQmmTransposedKernel");
- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+ let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
- let had_buf = hadamard_factors.map(|f| alloc_buffer_with_data::<B, i32>(&context, f));
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));
+ let had_buf = hadamard_factors.map(|f| alloc_allocation_with_data::<B, i32>(&context, f));
- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+ let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);
+ let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
@@ -219,9 +221,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
- let y_len = (input.m as usize) * (input.n as usize);
- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+ allocation_to_vec(&y_buf)
}
fn check_against_expected<T: ArrayElement + Float + Debug + Display>(
@@ -660,14 +660,9 @@ fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(
) {
// Use the dispatcher-selected tile configs: BM=8 small tile for M<48, big tile for M>=48.
// We always use the BM=8 small tile kernel for M in 4..=47 to match the dispatch threshold.
- let block_size: usize = if bits == 4 {
- 512
- } else {
- 256
- };
+ let block_size: usize = if bits == 4 { 512 } else { 256 };
- for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336])
- {
+ for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {
if n % 32 != 0 || k % block_size != 0 {
continue;
}
@@ -699,33 +694,29 @@ fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(
)
.unwrap();
- let w_buf = alloc_buffer_with_data::<B, u32>(
+ let w_buf = alloc_allocation_with_data::<B, u32>(
context,
&(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),
);
- let scales_buf = alloc_buffer_with_data::<B, T>(
+ let scales_buf = alloc_allocation_with_data::<B, T>(
context,
&(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
);
- let x_buf = alloc_buffer_with_data::<B, T>(
+ let x_buf = alloc_allocation_with_data::<B, T>(
context,
&(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),
);
- let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();
+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);
- let zp_stride = if bits == 4 {
- (num_groups + 1) / 2
- } else {
- num_groups
- };
+ let zp_stride = if bits == 4 { (num_groups + 1) / 2 } else { num_groups };
let zp_buf = use_zero_points.then(|| {
- alloc_buffer_with_data::<B, u8>(
+ alloc_allocation_with_data::<B, u8>(
context,
&(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),
)
});
let bias_buf = use_mlx_quant.then(|| {
- alloc_buffer_with_data::<B, T>(
+ alloc_allocation_with_data::<B, T>(
context,
&(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),
)
@@ -744,7 +735,7 @@ fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(
bias_buf.as_ref(),
&x_buf,
&mut y_buf,
- None::<&B::DenseBuffer>,
+ None::<&Allocation<B>>,
k as u32,
n as u32,
m as u32,
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index 034482e75..c2974c4d7 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -4,7 +4,7 @@ use backend_uzu::{
ArrayElement, DataType,
backends::{
common::{
- Backend, Context, DenseBuffer, Encoder, Kernels,
+ Allocation, Backend, Context, Encoder, Kernels,
kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},
},
cpu::Cpu,
@@ -17,20 +17,22 @@ use rand::{RngExt, SeedableRng, rngs::SmallRng};
use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
use crate::{
- common::{helpers::alloc_buffer_with_data, type_short_name},
+ common::{
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ type_short_name,
+ },
uzu_bench, uzu_test,
};
fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");
- let w_buf = alloc_buffer_with_data::<Cpu, u32>(&context, &input.w_packed);
- let scales_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.scales);
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<Cpu, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<Cpu, T>(&context, b));
- let x_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.x);
- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+ let w_buf = alloc_allocation_with_data::<Cpu, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.scales);
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<Cpu, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<Cpu, T>(&context, b));
+ let x_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.x);
+ let mut y_buf = alloc_allocation::<Cpu, T>(&context, (input.m as usize) * (input.n as usize));
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -58,21 +60,18 @@ fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
- let y_len = (input.m as usize) * (input.n as usize);
- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+ allocation_to_vec(&y_buf)
}
fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+ let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));
+ let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);
+ let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -93,7 +92,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
bias_buf.as_ref(),
&x_buf,
&mut y_buf,
- None::<&B::DenseBuffer>,
+ None::<&Allocation<B>>,
input.k,
input.n,
input.m,
@@ -102,9 +101,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
- let y_len = (input.m as usize) * (input.n as usize);
- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+ allocation_to_vec(&y_buf)
}
fn get_test_data<T: ArrayElement + Float>(
@@ -347,25 +344,25 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
)
.unwrap();
- let w_buf = alloc_buffer_with_data::<B, u32>(
+ let w_buf = alloc_allocation_with_data::<B, u32>(
context,
&gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),
);
- let scales_buf = alloc_buffer_with_data::<B, T>(
+ let scales_buf = alloc_allocation_with_data::<B, T>(
context,
&gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)
.iter()
.map(|&v| T::from(v).unwrap())
.collect::<Vec<_>>(),
);
- let x_buf = alloc_buffer_with_data::<B, T>(
+ let x_buf = alloc_allocation_with_data::<B, T>(
context,
&gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k)
.iter()
.map(|&v| T::from(v).unwrap())
.collect::<Vec<_>>(),
);
- let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();
+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);
let zp_buf = use_zero_points.then(|| {
let zp_stride = if bits == 4 {
@@ -373,10 +370,10 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
} else {
num_groups
};
- alloc_buffer_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))
+ alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))
});
let bias_buf = use_mlx_quant.then(|| {
- alloc_buffer_with_data::<B, T>(
+ alloc_allocation_with_data::<B, T>(
context,
&gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)
.iter()
@@ -397,7 +394,7 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
bias_buf.as_ref(),
&x_buf,
&mut y_buf,
- None::<&B::DenseBuffer>,
+ None::<&Allocation<B>>,
k as u32,
n as u32,
m as u32,
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
index fd448a593..80aa4f71d 100644
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
@@ -3,27 +3,29 @@ use std::fmt::{Debug, Display};
use backend_uzu::{
ArrayElement, DataType,
backends::{
- common::{Backend, Context, DenseBuffer, Encoder, Kernels, kernel::QuantizedMatmulQmvKernel},
+ common::{Backend, Context, Encoder, Kernels, kernel::QuantizedMatmulQmvKernel},
cpu::Cpu,
},
};
use num_traits::Float;
use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
+use crate::{
+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
+};
fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = B::Context::new().expect("Failed to create Context");
- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+ let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));
- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+ let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);
+ let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
@@ -51,9 +53,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
- let y_len = (input.m as usize) * (input.n as usize);
- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+ allocation_to_vec(&y_buf)
}
/// Create test data for a qmv scenario: m×k * k×n = m×n
diff --git a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs
index 3cd5d2648..63bb22b79 100644
--- a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs
@@ -1,13 +1,12 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
time::{Duration, Instant},
};
use backend_uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{Backend, Context, DenseBuffer, Encoder, Kernels, kernel::RMSNormKernel},
+ common::{Allocation, Context, Encoder, Kernels, kernel::RMSNormKernel, Backend},
cpu::Cpu,
},
};
@@ -138,25 +137,23 @@ fn get_output<
.expect("Failed to create RMSNormKernel");
let input_size = input.input.len();
- let input_array = context.create_array_from(&[input_size], &input.input, "");
- let input_array_buffer_rc = input_array.buffer();
- let input_array_borrow = input_array_buffer_rc.borrow();
- let input_array_deref = input_array_borrow.deref();
- let input_buffer = (!input.in_place).then(|| input_array_deref);
-
- let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");
- let output_array = match input.in_place {
- true => context.create_array_from(&[input_size], &input.output, ""),
- false => context.create_array_uninitialized(&[input_size], OutputT::data_type(), ""),
+ let input_buffer = (!input.in_place).then(|| context.create_array_from(&[input_size], &input.input).into_allocation());
+
+ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales);
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[input_size], &input.output).into_allocation(),
+ false => context
+ .create_array_uninitialized(&[input_size], OutputT::data_type())
+ .into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_buffer,
- scales_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- None::<&mut B::DenseBuffer>,
- None::<&B::DenseBuffer>,
+ input_buffer.as_ref(),
+ scales_array.allocation(),
+ &mut output,
+ None::<&mut Allocation<B>>,
+ None::<&Allocation<B>>,
input.batch_size,
input.element_count,
input.epsilon,
@@ -169,7 +166,11 @@ fn get_output<
let host_elapsed_ms = instant.elapsed().as_secs_f64() * 1e3;
let gpu_elapsed_ms = completed.gpu_execution_time();
- (output_array.as_slice().to_vec(), host_elapsed_ms, gpu_elapsed_ms)
+ (
+ crate::common::helpers::allocation_to_vec(&output),
+ host_elapsed_ms,
+ gpu_elapsed_ms,
+ )
}
fn test_internal<
@@ -430,19 +431,9 @@ fn bench_rms_norm(c: &mut Criterion) {
let (input_data, scale_data) = get_rms_norm_data(1337, batch_size, model_dim);
let input_size = batch_size * model_dim;
- let input_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();
- unsafe {
- std::slice::from_raw_parts_mut::<T>(input_buffer.cpu_ptr().as_ptr() as *mut T, input_size)
- .copy_from_slice(input_data.as_ref());
- }
-
- let scales_buffer = context.create_buffer(model_dim * std::mem::size_of::<T>()).unwrap();
- unsafe {
- std::slice::from_raw_parts_mut::<T>(scales_buffer.cpu_ptr().as_ptr() as *mut T, model_dim)
- .copy_from_slice(scale_data.as_ref());
- }
-
- let mut output_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();
+ let input_buffer = context.create_array_from(&[input_size], input_data.as_ref()).into_allocation();
+ let scales_buffer = context.create_array_from(&[model_dim], scale_data.as_ref()).into_allocation();
+ let mut output_buffer = context.create_array_uninitialized(&[input_size], T::data_type()).into_allocation();
group.throughput(Throughput::Elements((batch_size * model_dim) as u64));
@@ -455,8 +446,8 @@ fn bench_rms_norm(c: &mut Criterion) {
Some(&input_buffer),
&scales_buffer,
&mut output_buffer,
- None::<&mut <B as Backend>::DenseBuffer>,
- None::<&<B as Backend>::DenseBuffer>,
+ None::<&mut Allocation<B>>,
+ None::<&Allocation<B>>,
batch_size as u32,
model_dim as u32,
epsilon,
diff --git a/crates/backend-uzu/tests/unit/kernel/rope_test.rs b/crates/backend-uzu/tests/unit/kernel/rope_test.rs
index 013212eab..e30d780f6 100644
--- a/crates/backend-uzu/tests/unit/kernel/rope_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/rope_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -83,21 +82,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let queries_len = input.num_heads as usize * input.suffix_length as usize * input.head_dim as usize;
let keys_len = input.num_groups as usize * input.suffix_length as usize * input.head_dim as usize;
- let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");
- let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines, "");
- let sines_array = context.create_array_from(&[cos_sin_len], &input.sines, "");
- let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions, "");
- let rotated_queries_array = context.create_array_uninitialized(&[queries_len], T::data_type(), "");
- let rotated_keys_array = context.create_array_uninitialized(&[keys_len], T::data_type(), "");
+ let qkv_array = context.create_array_from(&[qkv_len], &input.qkv);
+ let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines);
+ let sines_array = context.create_array_from(&[cos_sin_len], &input.sines);
+ let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions);
+ let mut rotated_queries = context
+ .create_array_uninitialized(&[queries_len], T::data_type())
+ .into_allocation();
+ let mut rotated_keys = context
+ .create_array_uninitialized(&[keys_len], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- qkv_array.buffer().borrow().deref(),
- cosines_array.buffer().borrow().deref(),
- sines_array.buffer().borrow().deref(),
- token_positions_array.buffer().borrow().deref(),
- rotated_queries_array.buffer().borrow_mut().deref_mut(),
- rotated_keys_array.buffer().borrow_mut().deref_mut(),
+ qkv_array.allocation(),
+ cosines_array.allocation(),
+ sines_array.allocation(),
+ token_positions_array.allocation(),
+ &mut rotated_queries,
+ &mut rotated_keys,
input.head_dim,
input.rope_dim,
input.num_heads,
@@ -108,7 +111,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (rotated_queries_array.as_slice().to_vec(), rotated_keys_array.as_slice().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&rotated_queries),
+ crate::common::helpers::allocation_to_vec(&rotated_keys),
+ )
}
fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>) {
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rs b/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rs
index 2706fee92..3790951f6 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rs
@@ -1,7 +1,6 @@
use std::{
fmt::Display,
mem::size_of,
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -69,13 +68,13 @@ fn get_output_single<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> V
.expect("Failed to create ArgmaxSingleKernel");
let len = (input.batch_size * input.vocab_size) as usize;
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let output_array = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32, "");
+ let logits_array = context.create_array_from(&[len], &input.logits);
+ let mut output = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- logits_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ logits_array.allocation(),
+ &mut output,
input.batch_size,
input.vocab_size,
&mut encoder,
@@ -83,7 +82,7 @@ fn get_output_single<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> V
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice::<u32>().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<u32> {
@@ -95,29 +94,29 @@ fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) ->
.expect("Failed to create ArgmaxFinalKernel");
let len = (input.batch_size * input.vocab_size) as usize;
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let output_array = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32, "");
+ let logits_array = context.create_array_from(&[len], &input.logits);
+ let mut output = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32).into_allocation();
let block_size = 1024;
let grain_size = 4;
let elements_per_group = block_size * grain_size;
let vocab_groups_per_batch = (input.vocab_size as usize + elements_per_group - 1) / elements_per_group;
let partial_results_count = input.batch_size as usize * vocab_groups_per_batch;
- let mut partial_results_buffer = context
- .create_buffer(partial_results_count * size_of::<ArgmaxPair>())
- .expect("Failed to create partial results buffer");
+ let mut partial_results = context
+ .create_array_uninitialized(&[partial_results_count * size_of::<ArgmaxPair>()], DataType::U8)
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
main_kernel.encode(
- logits_array.buffer().borrow().deref(),
- &mut partial_results_buffer,
+ logits_array.allocation(),
+ &mut partial_results,
input.batch_size,
input.vocab_size,
&mut encoder,
);
final_kernel.encode(
- &partial_results_buffer,
- output_array.buffer().borrow_mut().deref_mut(),
+ &partial_results,
+ &mut output,
input.batch_size,
input.vocab_size,
&mut encoder,
@@ -125,7 +124,7 @@ fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) ->
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice::<u32>().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_single_pass<T: ArrayElement + Float + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rs
index f1fbc901f..3a227373e 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -133,29 +132,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let len = (input.batch_size * input.vocab_size) as usize;
let bitmask_len = input.bitmask.len();
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let bitmask_array = context.create_array_from(&[bitmask_len], &input.bitmask, "");
- let logits_array_buffer_rc = logits_array.buffer();
- let logits_array_borrow = logits_array_buffer_rc.borrow();
- let logits_array_deref = logits_array_borrow.deref();
- let logits_buffer = (!input.in_place).then(|| logits_array_deref);
- let output_array = match input.in_place {
- true => context.create_array_from(&[len], &input.logits, ""),
- false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());
+ let bitmask_array = context.create_array_from(&[bitmask_len], &input.bitmask);
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits).into_allocation(),
+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- logits_buffer,
- bitmask_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ logits_buffer.as_ref(),
+ bitmask_array.allocation(),
+ &mut output,
input.batch_size,
input.vocab_size,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_basic<T: ArrayElement + Float + Debug + Display>(in_place: bool) {
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rs
index 84e1b2b0d..c51363f0c 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -57,25 +56,21 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.expect("Failed to create GumbelKernel");
let len = (input.batch_size * input.vocab_size) as usize;
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let logits_array_buffer_rc = logits_array.buffer();
- let logits_array_borrow = logits_array_buffer_rc.borrow();
- let logits_array_deref = logits_array_borrow.deref();
- let logits_buffer = (!input.in_place).then(|| logits_array_deref);
+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());
- let seeds_array = context.create_array_from(&[input.batch_size as usize], &input.seeds, "");
+ let seeds_array = context.create_array_from(&[input.batch_size as usize], &input.seeds);
- let output_array = match input.in_place {
- true => context.create_array_from(&[len], &input.logits, ""),
- false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits).into_allocation(),
+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- logits_buffer,
- seeds_array.buffer().borrow().deref(),
+ logits_buffer.as_ref(),
+ seeds_array.allocation(),
0,
- output_array.buffer().borrow_mut().deref_mut(),
+ &mut output,
input.batch_size,
input.vocab_size,
&mut encoder,
@@ -83,7 +78,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
// Verify determinism: same seeds produce same output.
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rs
index 6f31a8411..f6d37afbd 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -51,20 +50,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.expect("Failed to create MinPKernel");
let len = (input.batch_size * input.vocab_size) as usize;
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let logits_array_buffer_rc = logits_array.buffer();
- let logits_array_borrow = logits_array_buffer_rc.borrow();
- let logits_array_deref = logits_array_borrow.deref();
- let logits_buffer = (!input.in_place).then(|| logits_array_deref);
- let output_array = match input.in_place {
- true => context.create_array_from(&[len], &input.logits, ""),
- false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits).into_allocation(),
+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- logits_buffer,
- output_array.buffer().borrow_mut().deref_mut(),
+ logits_buffer.as_ref(),
+ &mut output,
input.batch_size,
input.vocab_size,
input.min_p,
@@ -72,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_internal<T: ArrayElement + Float + Debug + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rs
index 544d32109..1ee3ad552 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -55,20 +54,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.expect("Failed to create TemperatureKernel");
let len = (input.batch_size * input.vocab_size) as usize;
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let logits_array_buffer_rc = logits_array.buffer();
- let logits_array_borrow = logits_array_buffer_rc.borrow();
- let logits_array_deref = logits_array_borrow.deref();
- let logits_buffer = (!input.in_place).then(|| logits_array_deref);
- let output_array = match input.in_place {
- true => context.create_array_from(&[len], &input.logits, ""),
- false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ let input_logits = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits).into_allocation(),
+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- logits_buffer,
- output_array.buffer().borrow_mut().deref_mut(),
+ input_logits.as_ref(),
+ &mut output,
input.batch_size,
input.vocab_size,
input.temperature,
@@ -77,7 +72,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn test_internal<T: ArrayElement + Float + Debug + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
index 597013648..07700d020 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs
@@ -1,18 +1,18 @@
-use std::mem::MaybeUninit;
+use std::mem::{MaybeUninit, size_of};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use num_traits::{Float, NumCast};
+use proptest::prelude::*;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
+use rand_distr::Normal;
use backend_uzu::{
- ArrayElement,
+ ArrayContextExt, ArrayElement, DataType,
backends::common::{
- Backend, Context, DenseBuffer, Encoder, Kernels,
+ AllocationType, Backend, Context, Encoder, Kernels,
gpu_types::ArgmaxPair,
kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},
},
};
-use criterion::{BenchmarkId, Criterion, Throughput};
-use num_traits::{Float, NumCast};
-use proptest::prelude::*;
-use rand::{RngExt, SeedableRng, rngs::SmallRng};
-use rand_distr::Normal;
use crate::{
common::{
@@ -43,17 +43,15 @@ fn do_argmax_backend<B: Backend, T: ArrayElement + Float>(
batch_size: usize,
vocab_size: usize,
) -> Result<ArgmaxTestResults, TestCaseError> {
- let logits_buffer = context.create_buffer(logits.len() * std::mem::size_of::<T>()).unwrap();
- unsafe {
- std::slice::from_raw_parts_mut::<T>(logits_buffer.cpu_ptr().as_ptr() as *mut T, logits.len())
- .copy_from_slice(logits);
- }
-
- let mut twopass_partial_results_buffer =
- context.create_buffer(batch_size * vocab_size.div_ceil(4096) * std::mem::size_of::<ArgmaxPair>()).unwrap();
-
- let mut single_output_buffer = context.create_buffer(batch_size * 4).unwrap();
- let mut twopass_output_buffer = context.create_buffer(batch_size * 4).unwrap();
+ let logits_buffer = context.create_array_from(&[logits.len()], logits).into_allocation();
+ let mut twopass_partial_results_buffer = context
+ .create_allocation(
+ batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),
+ AllocationType::Global,
+ )
+ .unwrap();
+ let mut single_output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();
+ let mut twopass_output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();
let single_kernel = <B::Kernels as Kernels>::ArgmaxSingleKernel::new(context, T::data_type()).unwrap();
let twopass_kernel_main = <B::Kernels as Kernels>::ArgmaxMainKernel::new(context, T::data_type()).unwrap();
@@ -77,23 +75,12 @@ fn do_argmax_backend<B: Backend, T: ArrayElement + Float>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let single_output = unsafe {
- std::slice::from_raw_parts(single_output_buffer.cpu_ptr().as_ptr() as *const u32, batch_size)
- .iter()
- .copied()
- .collect()
- };
-
- let twopass_output = unsafe {
- std::slice::from_raw_parts(twopass_output_buffer.cpu_ptr().as_ptr() as *const u32, batch_size)
- .iter()
- .copied()
- .collect()
- };
+ let single_output = crate::common::helpers::allocation_to_vec::<B, u32>(&single_output_buffer);
+ let twopass_output = crate::common::helpers::allocation_to_vec::<B, u32>(&twopass_output_buffer);
prop_assert_eq!(&single_output, &twopass_output, "single and twopass argmax output differs");
- Ok(ArgmaxTestResults(single_output))
+ Ok(ArgmaxTestResults(single_output.into_boxed_slice()))
}
fn get_argmax_data<T: ArrayElement + Float>(
@@ -143,7 +130,7 @@ fn bench_argmax(c: &mut Criterion) {
<<B as Backend>::Kernels as Kernels>::ArgmaxMainKernel::new(&context, T::data_type()).unwrap();
let twopass_kernel_final = <<B as Backend>::Kernels as Kernels>::ArgmaxFinalKernel::new(&context).unwrap();
- let mut output_buffer = context.create_buffer(batch_size * std::mem::size_of::<u32>()).unwrap();
+ let mut output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();
let mut group = c.benchmark_group(format!("{}/Kernel/Sampling/Argmax", type_short_name::<B>()));
@@ -154,14 +141,12 @@ fn bench_argmax(c: &mut Criterion) {
262144, // Gemma-3-1b
] {
let logits_data = get_argmax_data::<T>(1337, batch_size, vocab_size);
- let logits_buffer = context.create_buffer(logits_data.len() * std::mem::size_of::<T>()).unwrap();
- unsafe {
- std::slice::from_raw_parts_mut::<T>(logits_buffer.cpu_ptr().as_ptr() as *mut T, logits_data.len())
- .copy_from_slice(logits_data.as_ref());
- }
-
+ let logits_buffer = context.create_array_from(&[logits_data.len()], logits_data.as_ref()).into_allocation();
let mut twopass_partial_results_buffer = context
- .create_buffer(batch_size * vocab_size.div_ceil(4096) * std::mem::size_of::<ArgmaxPair>())
+ .create_allocation(
+ batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),
+ AllocationType::Global,
+ )
.unwrap();
group.throughput(Throughput::Elements(vocab_size as u64));
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rs
index 963784616..d5c800731 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -57,20 +56,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.expect("Failed to create TopKKernel");
let len = (input.batch_size * input.vocab_size) as usize;
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let logits_array_buffer_rc = logits_array.buffer();
- let logits_array_borrow = logits_array_buffer_rc.borrow();
- let logits_array_deref = logits_array_borrow.deref();
- let logits_buffer = (!input.in_place).then(|| logits_array_deref);
- let output_array = match input.in_place {
- true => context.create_array_from(&[len], &input.logits, ""),
- false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits).into_allocation(),
+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- logits_buffer,
- output_array.buffer().borrow_mut().deref_mut(),
+ logits_buffer.as_ref(),
+ &mut output,
input.batch_size,
input.vocab_size,
input.top_k,
@@ -79,7 +74,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn assert_top_k_equal<T: ArrayElement + Float + Display>(
diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rs
index ceb31bb3a..a5dbf16da 100644
--- a/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rs
@@ -1,25 +1,27 @@
use std::{
collections::HashSet,
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
+use half::{bf16, f16};
+use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::StdRng, seq::SliceRandom};
use backend_uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
common::{
- Backend, Context, Encoder, Kernels,
+ AllocationType, Backend, Context, Encoder, Kernels,
kernel::{TopPKernel, sampling::SamplingKernel},
},
cpu::Cpu,
},
session::parameter::{SamplingMethod, SamplingProcessingOrder},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use rand::{RngExt, SeedableRng, rngs::StdRng, seq::SliceRandom};
-use crate::uzu_test;
+use crate::{
+ common::helpers::{alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
+};
const TEST_SAMPLING_SEED: u64 = 42;
@@ -64,20 +66,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.expect("Failed to create TopPKernel");
let len = (input.batch_size * input.vocab_size) as usize;
- let logits_array = context.create_array_from(&[len], &input.logits, "");
- let logits_array_buffer_rc = logits_array.buffer();
- let logits_array_borrow = logits_array_buffer_rc.borrow();
- let logits_array_deref = logits_array_borrow.deref();
- let logits_buffer = (!input.in_place).then(|| logits_array_deref);
- let output_array = match input.in_place {
- true => context.create_array_from(&[len], &input.logits, ""),
- false => context.create_array_uninitialized(&[len], T::data_type(), ""),
+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());
+ let mut output = match input.in_place {
+ true => context.create_array_from(&[len], &input.logits).into_allocation(),
+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- logits_buffer,
- output_array.buffer().borrow_mut().deref_mut(),
+ logits_buffer.as_ref(),
+ &mut output,
input.batch_size,
input.vocab_size,
input.top_p,
@@ -86,7 +84,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&output)
}
fn assert_top_p_equal<T: ArrayElement + Float + Display>(
@@ -239,8 +237,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
let p = (k as f32) * 0.1;
- let kernel = SamplingKernel::<B>::new(&context, DataType::F32, batch_size, vocab_size)
- .expect("Failed to create sampling kernel");
+ let kernel = SamplingKernel::<B>::new(&context, DataType::F32).expect("Failed to create sampling kernel");
// Build probability table
let low_prob = (1.0 - p) / (vocab_size - k) as f32;
@@ -271,25 +268,25 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
})
.collect();
- let output_array = context.create_array_uninitialized(&[batch_size], DataType::U32, "");
+ let mut output_allocation = context
+ .create_allocation(batch_size * std::mem::size_of::<u32>(), AllocationType::Global)
+ .expect("Failed to create allocation");
let num_samples = 1000;
let mut counter = vec![0i32; batch_size * vocab_size];
for draw in 0..num_samples {
- let logits_array = context.create_array_from(&[batch_size * vocab_size], &logits, "");
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
- let seeds_array = context.create_array_from(&[batch_size], &seeds, "");
+ let mut logits_allocation = alloc_allocation_with_data::<B, f32>(&context, &logits);
+ let seeds_allocation = alloc_allocation_with_data::<B, u64>(&context, &seeds);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
- logits_array.buffer().borrow_mut().deref_mut(),
- seeds_array.buffer().borrow().deref(),
- 0,
+ &mut logits_allocation,
+ &seeds_allocation,
None,
- 0,
- output_array.buffer().borrow_mut().deref_mut(),
+ &mut output_allocation,
SamplingMethod::Stochastic {
temperature: None,
top_k: None,
@@ -304,7 +301,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
.expect("encode");
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let sampled_ids: &[u32] = output_array.as_slice();
+ let sampled_ids = allocation_to_vec::<B, u32>(&output_allocation);
for (i, &sampled_id) in sampled_ids.iter().enumerate() {
assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");
@@ -361,7 +358,7 @@ fn test_topp_sampling_statistical_large() {
let context = <B as Backend>::Context::new().expect("Failed to create Context");
let kernel =
- SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
+ SamplingKernel::<B>::new(&context, DataType::F32).expect("Failed to create sampling kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -377,24 +374,24 @@ fn test_topp_sampling_statistical_large() {
probs[b * VOCAB..(b + 1) * VOCAB].copy_from_slice(&dist);
}
- let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");
+ let mut output_allocation = context
+ .create_allocation(BATCH * std::mem::size_of::<u32>(), AllocationType::Global)
+ .expect("Failed to create allocation");
let mut counters = vec![0u32; BATCH * VOCAB];
for draw in 0..NUM_DRAWS {
- let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];
- let seeds_array = context.create_array_from(&[BATCH], &seeds, "");
+ let mut logits_allocation = alloc_allocation_with_data::<B, f32>(&context, &logits);
+ let seeds_allocation = alloc_allocation_with_data::<B, u64>(&context, &seeds);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
- logits_array.buffer().borrow_mut().deref_mut(),
- seeds_array.buffer().borrow().deref(),
- 0,
+ &mut logits_allocation,
+ &seeds_allocation,
None,
- 0,
- output_array.buffer().borrow_mut().deref_mut(),
+ &mut output_allocation,
SamplingMethod::Stochastic {
temperature: None,
top_k: None,
@@ -409,7 +406,7 @@ fn test_topp_sampling_statistical_large() {
.expect("encode");
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let sample_ids: &[u32] = output_array.as_slice();
+ let sample_ids = allocation_to_vec::<B, u32>(&output_allocation);
for (b, &tok) in sample_ids.iter().enumerate() {
counters[b * VOCAB + tok as usize] += 1;
}
@@ -449,7 +446,7 @@ fn perf_topp_128k_vocab() {
let context = <B as Backend>::Context::new().expect("Failed to create Context");
let kernel =
- SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");
+ SamplingKernel::<B>::new(&context, DataType::F32).expect("Failed to create sampling kernel");
let mut rng = StdRng::seed_from_u64(123);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -457,21 +454,21 @@ fn perf_topp_128k_vocab() {
*x = rng.random_range(-6.0f32..6.0f32);
}
- let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];
- let seeds_array = context.create_array_from(&[BATCH], &seeds, "");
- let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");
+ let mut logits_allocation = alloc_allocation_with_data::<B, f32>(&context, &logits);
+ let seeds_allocation = alloc_allocation_with_data::<B, u64>(&context, &seeds);
+ let mut output_allocation = context
+ .create_allocation(BATCH * std::mem::size_of::<u32>(), AllocationType::Global)
+ .expect("Failed to create allocation");
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
- logits_array.buffer().borrow_mut().deref_mut(),
- seeds_array.buffer().borrow().deref(),
- 0,
+ &mut logits_allocation,
+ &seeds_allocation,
None,
- 0,
- output_array.buffer().borrow_mut().deref_mut(),
+ &mut output_allocation,
SamplingMethod::Stochastic {
temperature: None,
top_k: None,
@@ -499,8 +496,8 @@ fn perf_topp_128k_vocab() {
host_elapsed_ms
);
- let sample_ids: &[u32] = output_array.as_slice();
- for &tok in sample_ids {
+ let sample_ids = allocation_to_vec::<B, u32>(&output_allocation);
+ for tok in sample_ids {
assert!((tok as usize) < VOCAB, "Sampled id out of range");
}
});
diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs
index 343ca94f5..47da37472 100644
--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -42,46 +41,35 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
)
.expect("Failed to create ShortConvDecodeKernel");
- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");
- let w_array = context.create_array_from(&[input.w.len()], &input.w, "");
- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, ""));
+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);
+ let w_array = context.create_array_from(&[input.w.len()], &input.w);
+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));
let out_size = input.suffix_len as usize * input.model_dim as usize;
- let out_array = context.create_array_uninitialized(&[out_size], T::data_type(), "");
+ let mut out = context.create_array_uninitialized(&[out_size], T::data_type()).into_allocation();
let state_size = input.model_dim as usize * input.state_stride as usize;
- let next_state_array = if state_in_place {
- if state_size > 0 {
- context.create_array_from(&[state_size], &input.state, "")
- } else {
- context.create_array_uninitialized(&[1], T::data_type(), "")
- }
- } else {
- context.create_array_uninitialized(&[state_size.max(1)], T::data_type(), "")
- };
+ let state_allocation_size = state_size.max(1);
+ let state_data: Vec<T> =
+ input.state.iter().copied().chain(std::iter::repeat(T::zero())).take(state_allocation_size).collect();
- let state_array = if state_in_place {
- None
- } else if state_size > 0 {
- Some(context.create_array_from(&[state_size], &input.state, ""))
+ let mut next_state = if state_in_place {
+ context.create_array_from(&[state_allocation_size], &state_data).into_allocation()
} else {
- Some(context.create_array_uninitialized(&[1], T::data_type(), ""))
+ context.create_array_uninitialized(&[state_allocation_size], T::data_type()).into_allocation()
};
- let bias_buf_rc = b_array.as_ref().map(|b| b.buffer());
- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
-
- let state_buf_rc = state_array.as_ref().map(|s| s.buffer());
- let state_buf_borrow = state_buf_rc.as_ref().map(|rc| rc.borrow());
+ let state_array =
+ (!state_in_place).then(|| context.create_array_from(&[state_allocation_size], &state_data));
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- in_proj_array.buffer().borrow().deref(),
- w_array.buffer().borrow().deref(),
- bias_buf_borrow.as_deref(),
- state_buf_borrow.as_deref(),
- out_array.buffer().borrow_mut().deref_mut(),
- next_state_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.allocation(),
+ w_array.allocation(),
+ b_array.as_ref().map(|bias| bias.allocation()),
+ state_array.as_ref().map(|state| state.allocation()),
+ &mut out,
+ &mut next_state,
input.suffix_len,
input.kernel_size,
input.in_proj_stride,
@@ -91,7 +79,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (out_array.as_slice().to_vec(), next_state_array.as_slice().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&out),
+ crate::common::helpers::allocation_to_vec(&next_state),
+ )
}
fn get_test_data_basic<T: ArrayElement + Float>(
diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs
index 0bfb0e1ed..d9c935afc 100644
--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -30,18 +29,20 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(&context, T::data_type())
.expect("Failed to create ShortConvPackKernel");
- let state_in_array = context.create_array_from(&[input.state_in.len()], &input.state_in, "");
- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");
+ let state_in_array = context.create_array_from(&[input.state_in.len()], &input.state_in);
+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);
let padded_rows = (input.state_stride + input.suffix_len) as usize;
let padded_size = padded_rows * input.model_dim as usize;
- let padded_array = context.create_array_uninitialized(&[padded_size], T::data_type(), "");
+ let mut padded = context
+ .create_array_uninitialized(&[padded_size], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- state_in_array.buffer().borrow().deref(),
- in_proj_array.buffer().borrow().deref(),
- padded_array.buffer().borrow_mut().deref_mut(),
+ state_in_array.allocation(),
+ in_proj_array.allocation(),
+ &mut padded,
input.state_stride,
input.suffix_len,
input.in_proj_stride,
@@ -50,7 +51,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- padded_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&padded)
}
/// Build test input for ShortConvPack.
diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs
index 45fd94d33..8723c762a 100644
--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -34,28 +33,29 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(&context, T::data_type(), has_bias)
.expect("Failed to create ShortConvPrefillKernel");
- let padded_array = context.create_array_from(&[input.padded.len()], &input.padded, "");
- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");
- let w_array = context.create_array_from(&[input.w.len()], &input.w, "");
- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, ""));
+ let padded_array = context.create_array_from(&[input.padded.len()], &input.padded);
+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);
+ let w_array = context.create_array_from(&[input.w.len()], &input.w);
+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));
let out_size = input.suffix_len as usize * input.model_dim as usize;
- let out_array = context.create_array_uninitialized(&[out_size], T::data_type(), "");
+ let mut out = context
+ .create_array_uninitialized(&[out_size], T::data_type())
+ .into_allocation();
let state_out_size = input.model_dim as usize * input.state_stride as usize;
- let state_out_array = context.create_array_uninitialized(&[state_out_size], T::data_type(), "");
-
- let bias_buf_rc = b_array.as_ref().map(|b| b.buffer());
- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
+ let mut state_out = context
+ .create_array_uninitialized(&[state_out_size], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- padded_array.buffer().borrow().deref(),
- in_proj_array.buffer().borrow().deref(),
- w_array.buffer().borrow().deref(),
- bias_buf_borrow.as_deref(),
- out_array.buffer().borrow_mut().deref_mut(),
- state_out_array.buffer().borrow_mut().deref_mut(),
+ padded_array.allocation(),
+ in_proj_array.allocation(),
+ w_array.allocation(),
+ b_array.as_ref().map(|bias| bias.allocation()),
+ &mut out,
+ &mut state_out,
input.suffix_len,
input.kernel_size,
input.in_proj_stride,
@@ -65,7 +65,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (out_array.as_slice().to_vec(), state_out_array.as_slice().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&out),
+ crate::common::helpers::allocation_to_vec(&state_out),
+ )
}
fn get_test_data_basic<T: ArrayElement + Float>(
diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs
index db7245033..c0a5bc778 100644
--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -32,44 +31,48 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let context = B::Context::new().expect("Failed to create Context");
let has_bias = input.b.is_some();
- let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(&context, T::data_type(), has_bias)
- .expect("Failed to create ShortConvTrieKernel");
+ let kernel =
+ <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(&context, T::data_type(), has_bias)
+ .expect("Failed to create ShortConvTrieKernel");
- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");
- let w_array = context.create_array_from(&[input.w.len()], &input.w, "");
- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, ""));
+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);
+ let w_array = context.create_array_from(&[input.w.len()], &input.w);
+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));
let out_size = input.suffix_len as usize * input.model_dim as usize;
- let out_array = context.create_array_uninitialized(&[out_size], T::data_type(), "");
+ let mut out = context.create_array_uninitialized(&[out_size], T::data_type()).into_allocation();
let state_stride = input.state_stride as usize;
let model_dim = input.model_dim as usize;
let suffix_len = input.suffix_len as usize;
let base_state_size = model_dim * state_stride;
- let base_state_array = if base_state_size > 0 {
- context.create_array_from(&[base_state_size], &input.base_state, "")
- } else {
- context.create_array_uninitialized(&[1], T::data_type(), "")
- };
-
- let parents_array = context.create_array_from(&[input.parents.len()], &input.parents, "");
+ let base_state_allocation_size = base_state_size.max(1);
+ let base_state_data: Vec<T> = input
+ .base_state
+ .iter()
+ .copied()
+ .chain(std::iter::repeat(T::zero()))
+ .take(base_state_allocation_size)
+ .collect();
+ let base_state_array = context.create_array_from(&[base_state_allocation_size], &base_state_data);
+
+ let parents_array = context.create_array_from(&[input.parents.len()], &input.parents);
let suffix_state_size = suffix_len * model_dim * state_stride;
- let suffix_state_array = context.create_array_uninitialized(&[suffix_state_size.max(1)], T::data_type(), "");
-
- let bias_buf_rc = b_array.as_ref().map(|b| b.buffer());
- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
+ let mut suffix_state = context
+ .create_array_uninitialized(&[suffix_state_size.max(1)], T::data_type())
+ .into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- in_proj_array.buffer().borrow().deref(),
- w_array.buffer().borrow().deref(),
- bias_buf_borrow.as_deref(),
- base_state_array.buffer().borrow().deref(),
- parents_array.buffer().borrow().deref(),
- out_array.buffer().borrow_mut().deref_mut(),
- suffix_state_array.buffer().borrow_mut().deref_mut(),
+ in_proj_array.allocation(),
+ w_array.allocation(),
+ b_array.as_ref().map(|bias| bias.allocation()),
+ base_state_array.allocation(),
+ parents_array.allocation(),
+ &mut out,
+ &mut suffix_state,
input.suffix_len,
input.kernel_size,
input.in_proj_stride,
@@ -79,7 +82,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (out_array.as_slice().to_vec(), suffix_state_array.as_slice().to_vec())
+ (
+ crate::common::helpers::allocation_to_vec(&out),
+ crate::common::helpers::allocation_to_vec(&suffix_state),
+ )
}
/// Linear chain: node 0 has parent -1 (root), node i has parent i-1.
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
index 0339d5d58..8bad6a293 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -101,31 +100,27 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
let c_out_size = input.suffix_len as usize * input.proj_dim as usize;
let state_size = input.num_channels as usize * input.state_stride as usize;
- let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");
- let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");
- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));
+ let x_array = context.create_array_from(&[input.x.len()], &input.x);
+ let w_array = context.create_array_from(&[input.w.len()], &input.w);
+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));
- let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");
- let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");
- let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");
-
- let b_buf = b_array.as_ref().map(|a| a.buffer());
- let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());
- let b_deref: Option<&B::DenseBuffer> = b_borrow.as_ref().map(|b| b.deref());
+ let mut x_out = context.create_array_uninitialized(&[x_out_size], T::data_type()).into_allocation();
+ let mut b_out = context.create_array_uninitialized(&[b_out_size], T::data_type()).into_allocation();
+ let mut c_out = context.create_array_uninitialized(&[c_out_size], T::data_type()).into_allocation();
if input.state_in_place {
- let next_state_array = context.create_array_from(&[state_size], &input.state, "next_state");
+ let mut next_state = context.create_array_from(&[state_size], &input.state).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- x_array.buffer().borrow().deref(),
- w_array.buffer().borrow().deref(),
- b_deref,
- None::<&B::DenseBuffer>,
- x_out_array.buffer().borrow_mut().deref_mut(),
- b_out_array.buffer().borrow_mut().deref_mut(),
- c_out_array.buffer().borrow_mut().deref_mut(),
- next_state_array.buffer().borrow_mut().deref_mut(),
+ x_array.allocation(),
+ w_array.allocation(),
+ b_array.as_ref().map(|bias| bias.allocation()),
+ None::<&backend_uzu::backends::common::Allocation<B>>,
+ &mut x_out,
+ &mut b_out,
+ &mut c_out,
+ &mut next_state,
input.kernel_size,
input.row_stride,
input.state_stride,
@@ -139,25 +134,26 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
Output {
- x_out: x_out_array.as_slice().to_vec(),
- b_out: b_out_array.as_slice().to_vec(),
- c_out: c_out_array.as_slice().to_vec(),
- next_state: next_state_array.as_slice().to_vec(),
+ x_out: crate::common::helpers::allocation_to_vec(&x_out),
+ b_out: crate::common::helpers::allocation_to_vec(&b_out),
+ c_out: crate::common::helpers::allocation_to_vec(&c_out),
+ next_state: crate::common::helpers::allocation_to_vec(&next_state),
}
} else {
- let state_array = context.create_array_from(&[state_size], &input.state, "state");
- let next_state_array = context.create_array_uninitialized(&[state_size], T::data_type(), "next_state");
+ let state_array = context.create_array_from(&[state_size], &input.state);
+ let mut next_state =
+ context.create_array_uninitialized(&[state_size], T::data_type()).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- x_array.buffer().borrow().deref(),
- w_array.buffer().borrow().deref(),
- b_deref,
- Some(state_array.buffer().borrow().deref()),
- x_out_array.buffer().borrow_mut().deref_mut(),
- b_out_array.buffer().borrow_mut().deref_mut(),
- c_out_array.buffer().borrow_mut().deref_mut(),
- next_state_array.buffer().borrow_mut().deref_mut(),
+ x_array.allocation(),
+ w_array.allocation(),
+ b_array.as_ref().map(|bias| bias.allocation()),
+ Some(state_array.allocation()),
+ &mut x_out,
+ &mut b_out,
+ &mut c_out,
+ &mut next_state,
input.kernel_size,
input.row_stride,
input.state_stride,
@@ -171,10 +167,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
Output {
- x_out: x_out_array.as_slice().to_vec(),
- b_out: b_out_array.as_slice().to_vec(),
- c_out: c_out_array.as_slice().to_vec(),
- next_state: next_state_array.as_slice().to_vec(),
+ x_out: crate::common::helpers::allocation_to_vec(&x_out),
+ b_out: crate::common::helpers::allocation_to_vec(&b_out),
+ c_out: crate::common::helpers::allocation_to_vec(&c_out),
+ next_state: crate::common::helpers::allocation_to_vec(&next_state),
}
}
}
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
index 502f49e77..1e2de9263 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -56,15 +55,15 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let total_rows = input.state_stride as usize + input.suffix_len as usize;
let padded_size = total_rows * input.row_stride as usize;
- let state_array = context.create_array_from(&[state_size], &input.state_in, "state_in");
- let x_array = context.create_array_from(&[x_size], &input.x, "x");
- let padded_array = context.create_array_uninitialized(&[padded_size], T::data_type(), "padded");
+ let state_array = context.create_array_from(&[state_size], &input.state_in);
+ let x_array = context.create_array_from(&[x_size], &input.x);
+ let mut padded = context.create_array_uninitialized(&[padded_size], T::data_type()).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- state_array.buffer().borrow().deref(),
- x_array.buffer().borrow().deref(),
- padded_array.buffer().borrow_mut().deref_mut(),
+ state_array.allocation(),
+ x_array.allocation(),
+ &mut padded,
input.state_stride,
input.row_stride,
input.suffix_len,
@@ -73,7 +72,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
- padded_array.as_slice().to_vec()
+ crate::common::helpers::allocation_to_vec(&padded)
}
fn get_test_data<T: ArrayElement + Float>(
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
index 99ccd937c..6be3cc6b2 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -90,28 +89,25 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
let c_out_size = input.suffix_len as usize * input.proj_dim as usize;
let state_size = input.num_channels as usize * input.state_stride as usize;
- let padded_array = context.create_array_from(&[input.padded.len()], &input.padded, "padded");
- let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");
- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));
+ let padded_array = context.create_array_from(&[input.padded.len()], &input.padded);
+ let w_array = context.create_array_from(&[input.w.len()], &input.w);
+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));
- let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");
- let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");
- let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");
- let state_out_array = context.create_array_uninitialized(&[state_size], T::data_type(), "state_out");
-
- let b_buf = b_array.as_ref().map(|a| a.buffer());
- let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());
- let b_deref: Option<&B::DenseBuffer> = b_borrow.as_ref().map(|b| b.deref());
+ let mut x_out = context.create_array_uninitialized(&[x_out_size], T::data_type()).into_allocation();
+ let mut b_out = context.create_array_uninitialized(&[b_out_size], T::data_type()).into_allocation();
+ let mut c_out = context.create_array_uninitialized(&[c_out_size], T::data_type()).into_allocation();
+ let mut state_out =
+ context.create_array_uninitialized(&[state_size], T::data_type()).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- padded_array.buffer().borrow().deref(),
- w_array.buffer().borrow().deref(),
- b_deref,
- x_out_array.buffer().borrow_mut().deref_mut(),
- b_out_array.buffer().borrow_mut().deref_mut(),
- c_out_array.buffer().borrow_mut().deref_mut(),
- state_out_array.buffer().borrow_mut().deref_mut(),
+ padded_array.allocation(),
+ w_array.allocation(),
+ b_array.as_ref().map(|bias| bias.allocation()),
+ &mut x_out,
+ &mut b_out,
+ &mut c_out,
+ &mut state_out,
input.suffix_len,
input.kernel_size,
input.row_stride,
@@ -125,10 +121,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
Output {
- x_out: x_out_array.as_slice().to_vec(),
- b_out: b_out_array.as_slice().to_vec(),
- c_out: c_out_array.as_slice().to_vec(),
- state_out: state_out_array.as_slice().to_vec(),
+ x_out: crate::common::helpers::allocation_to_vec(&x_out),
+ b_out: crate::common::helpers::allocation_to_vec(&b_out),
+ c_out: crate::common::helpers::allocation_to_vec(&c_out),
+ state_out: crate::common::helpers::allocation_to_vec(&state_out),
}
}
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
index c2bc85e19..0269536f6 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -102,14 +101,14 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
let total_x = input.suffix_len * input.num_heads * input.head_dim;
- let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");
- let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");
- let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");
- let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");
- let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");
- let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");
- let state_array = context.create_array_from(&[input.state.len()], &input.state, "state");
- let y_array = context.create_array_uninitialized(&[total_x], T::data_type(), "y");
+ let x_array = context.create_array_from(&[input.x.len()], &input.x);
+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt);
+ let b_array = context.create_array_from(&[input.b.len()], &input.b);
+ let c_array = context.create_array_from(&[input.c.len()], &input.c);
+ let d_array = context.create_array_from(&[input.d.len()], &input.d);
+ let z_array = context.create_array_from(&[input.z.len()], &input.z);
+ let mut state = context.create_array_from(&[input.state.len()], &input.state).into_allocation();
+ let mut y = context.create_array_uninitialized(&[total_x], T::data_type()).into_allocation();
let x_strides: [u32; 3] = input.x_strides.map(|s| s as u32);
let dt_strides: [u32; 2] = input.dt_strides.map(|s| s as u32);
@@ -122,14 +121,14 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(&context, T::data_type())
.expect("Failed to create SSDPrefillKernel");
kernel.encode(
- x_array.buffer().borrow().deref(),
- dt_array.buffer().borrow().deref(),
- b_array.buffer().borrow().deref(),
- c_array.buffer().borrow().deref(),
- d_array.buffer().borrow().deref(),
- z_array.buffer().borrow().deref(),
- state_array.buffer().borrow_mut().deref_mut(),
- y_array.buffer().borrow_mut().deref_mut(),
+ x_array.allocation(),
+ dt_array.allocation(),
+ b_array.allocation(),
+ c_array.allocation(),
+ d_array.allocation(),
+ z_array.allocation(),
+ &mut state,
+ &mut y,
input.suffix_len as u32,
input.group_size as u32,
input.state_dim as u32,
@@ -147,14 +146,14 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
<<B as Backend>::Kernels as Kernels>::SSDPrefillSequentialKernel::new(&context, T::data_type())
.expect("Failed to create SSDPrefillSequentialKernel");
kernel.encode(
- x_array.buffer().borrow().deref(),
- dt_array.buffer().borrow().deref(),
- b_array.buffer().borrow().deref(),
- c_array.buffer().borrow().deref(),
- d_array.buffer().borrow().deref(),
- z_array.buffer().borrow().deref(),
- state_array.buffer().borrow_mut().deref_mut(),
- y_array.buffer().borrow_mut().deref_mut(),
+ x_array.allocation(),
+ dt_array.allocation(),
+ b_array.allocation(),
+ c_array.allocation(),
+ d_array.allocation(),
+ z_array.allocation(),
+ &mut state,
+ &mut y,
input.suffix_len as u32,
input.group_size as u32,
input.state_dim as u32,
@@ -171,8 +170,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
Output {
- y: y_array.as_slice().to_vec(),
- state: state_array.as_slice().to_vec(),
+ y: crate::common::helpers::allocation_to_vec(&y),
+ state: crate::common::helpers::allocation_to_vec(&state),
}
}
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs
index b443d33c0..0d15b3a64 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -63,20 +62,21 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
let z_out_size = input.suffix_length as usize * input.inner_dim as usize;
let dt_out_size = input.suffix_length as usize * input.num_heads as usize;
- let input_array = context.create_array_from(&[input.input.len()], &input.input, "input");
- let z_bias_array = context.create_array_from(&[input.z_bias.len()], &input.z_bias, "z_bias");
+ let input_array = context.create_array_from(&[input.input.len()], &input.input);
+ let z_bias_array = context.create_array_from(&[input.z_bias.len()], &input.z_bias);
- let conv_out_array = context.create_array_uninitialized(&[conv_out_size], T::data_type(), "conv_out");
- let z_out_array = context.create_array_uninitialized(&[z_out_size], T::data_type(), "z_out");
- let dt_out_array = context.create_array_uninitialized(&[dt_out_size], T::data_type(), "dt_out");
+ let mut conv_out =
+ context.create_array_uninitialized(&[conv_out_size], T::data_type()).into_allocation();
+ let mut z_out = context.create_array_uninitialized(&[z_out_size], T::data_type()).into_allocation();
+ let mut dt_out = context.create_array_uninitialized(&[dt_out_size], T::data_type()).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_array.buffer().borrow().deref(),
- conv_out_array.buffer().borrow_mut().deref_mut(),
- z_out_array.buffer().borrow_mut().deref_mut(),
- dt_out_array.buffer().borrow_mut().deref_mut(),
- z_bias_array.buffer().borrow().deref(),
+ input_array.allocation(),
+ &mut conv_out,
+ &mut z_out,
+ &mut dt_out,
+ z_bias_array.allocation(),
input.suffix_length,
input.total_dim,
input.conv_dim,
@@ -87,9 +87,9 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
Output {
- conv_out: conv_out_array.as_slice().to_vec(),
- z_out: z_out_array.as_slice().to_vec(),
- dt_out: dt_out_array.as_slice().to_vec(),
+ conv_out: crate::common::helpers::allocation_to_vec(&conv_out),
+ z_out: crate::common::helpers::allocation_to_vec(&z_out),
+ dt_out: crate::common::helpers::allocation_to_vec(&dt_out),
}
}
diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs
index 35b4bc072..a71cc9955 100644
--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs
@@ -1,6 +1,5 @@
use std::{
fmt::{Debug, Display},
- ops::{Deref, DerefMut},
};
use backend_uzu::{
@@ -96,29 +95,29 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
let y_size = bsz * h * dh;
let ns_size = bsz * h * dh * n;
- let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");
- let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");
- let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");
- let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");
- let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");
- let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");
+ let x_array = context.create_array_from(&[input.x.len()], &input.x);
+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt);
+ let b_array = context.create_array_from(&[input.b.len()], &input.b);
+ let c_array = context.create_array_from(&[input.c.len()], &input.c);
+ let d_array = context.create_array_from(&[input.d.len()], &input.d);
+ let z_array = context.create_array_from(&[input.z.len()], &input.z);
- let y_array = context.create_array_uninitialized(&[y_size], T::data_type(), "y");
+ let mut y = context.create_array_uninitialized(&[y_size], T::data_type()).into_allocation();
if input.state_in_place {
- let ns_array = context.create_array_from(&[ns_size], &input.state, "next_state");
+ let mut next_state = context.create_array_from(&[ns_size], &input.state).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- x_array.buffer().borrow().deref(),
- dt_array.buffer().borrow().deref(),
- b_array.buffer().borrow().deref(),
- c_array.buffer().borrow().deref(),
- d_array.buffer().borrow().deref(),
- z_array.buffer().borrow().deref(),
- None::<&<B as Backend>::DenseBuffer>,
- y_array.buffer().borrow_mut().deref_mut(),
- ns_array.buffer().borrow_mut().deref_mut(),
+ x_array.allocation(),
+ dt_array.allocation(),
+ b_array.allocation(),
+ c_array.allocation(),
+ d_array.allocation(),
+ z_array.allocation(),
+ None::<&backend_uzu::backends::common::Allocation<B>>,
+ &mut y,
+ &mut next_state,
(h / g) as u32,
input.n,
&x_strides,
@@ -133,24 +132,25 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
Output {
- y: y_array.as_slice().to_vec(),
- next_state: ns_array.as_slice().to_vec(),
+ y: crate::common::helpers::allocation_to_vec(&y),
+ next_state: crate::common::helpers::allocation_to_vec(&next_state),
}
} else {
- let state_array = context.create_array_from(&[ns_size], &input.state, "state");
- let ns_array = context.create_array_uninitialized(&[ns_size], T::data_type(), "next_state");
+ let state_array = context.create_array_from(&[ns_size], &input.state);
+ let mut next_state =
+ context.create_array_uninitialized(&[ns_size], T::data_type()).into_allocation();
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- x_array.buffer().borrow().deref(),
- dt_array.buffer().borrow().deref(),
- b_array.buffer().borrow().deref(),
- c_array.buffer().borrow().deref(),
- d_array.buffer().borrow().deref(),
- z_array.buffer().borrow().deref(),
- Some(state_array.buffer().borrow().deref()),
- y_array.buffer().borrow_mut().deref_mut(),
- ns_array.buffer().borrow_mut().deref_mut(),
+ x_array.allocation(),
+ dt_array.allocation(),
+ b_array.allocation(),
+ c_array.allocation(),
+ d_array.allocation(),
+ z_array.allocation(),
+ Some(state_array.allocation()),
+ &mut y,
+ &mut next_state,
(h / g) as u32,
input.n,
&x_strides,
@@ -165,8 +165,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<T
encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
Output {
- y: y_array.as_slice().to_vec(),
- next_state: ns_array.as_slice().to_vec(),
+ y: crate::common::helpers::allocation_to_vec(&y),
+ next_state: crate::common::helpers::allocation_to_vec(&next_state),
}
}
}
diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs
index 6081d5817..7a499c6ab 100644
--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs
@@ -1,16 +1,16 @@
-use std::{
- fmt::Debug,
- ops::{Deref, DerefMut},
-};
+use std::fmt::Debug;
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
- ArrayContextExt, ArrayElement,
+ ArrayElement,
backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::uzu_test;
+use crate::{
+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
input: Box<[T]>,
@@ -64,30 +64,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
.expect("Failed to create TensorAddBiasKernel");
let size = input.length as usize;
- let input_array = context.create_array_from(&[size], &input.input, "");
- let input_array_buffer_rc = input_array.buffer();
- let input_array_borrow = input_array_buffer_rc.borrow();
- let input_array_deref = input_array_borrow.deref();
- let input_buffer = (!in_place).then(|| input_array_deref);
-
- let bias_array = context.create_array_from(&[input.num_cols as usize], &input.bias, "");
- let output_array = match in_place {
- true => context.create_array_from(&[size], &input.output, ""),
- false => context.create_array_uninitialized(&[size], T::data_type(), ""),
+ let input_allocation = (!in_place).then(|| alloc_allocation_with_data::<B, T>(&context, &input.input));
+ let bias_allocation = alloc_allocation_with_data::<B, T>(&context, &input.bias);
+ let mut output_allocation = match in_place {
+ true => alloc_allocation_with_data::<B, T>(&context, &input.output),
+ false => alloc_allocation::<B, T>(&context, size),
};
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_buffer,
- bias_array.buffer().borrow_mut().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ input_allocation.as_ref(),
+ &bias_allocation,
+ &mut output_allocation,
input.num_cols,
input.length,
&mut encoder,
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&output_allocation)
}
fn test<T: ArrayElement + Float + Debug>(in_place: bool) {
diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs
index 50d516aea..4355a7f56 100644
--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs
@@ -1,19 +1,22 @@
-use std::{
- fmt::{Debug, Display},
- ops::{Deref, DerefMut},
-};
+use std::fmt::{Debug, Display};
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
- ArrayContextExt, ArrayElement, DataType,
+ ArrayElement, DataType,
backends::{
common::{Backend, Context, Encoder, Kernels, kernel::TensorAddScaleKernel},
cpu::Cpu,
},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::{common::assert::assert_eq_float, uzu_test};
+use crate::{
+ common::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
input: Box<[T]>,
@@ -41,15 +44,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let length = input.length as usize;
let num_cols = input.num_cols as usize;
- let input_array = context.create_array_from(&[length], &input.input, "");
- let bias_array = context.create_array_from(&[num_cols], &input.bias, "");
- let output_array = context.create_array_uninitialized(&[length], T::data_type(), "");
+ let input_allocation = alloc_allocation_with_data::<B, T>(&context, &input.input[..length]);
+ let bias_allocation = alloc_allocation_with_data::<B, T>(&context, &input.bias[..num_cols]);
+ let mut output_allocation = alloc_allocation::<B, T>(&context, length);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
- input_array.buffer().borrow().deref(),
- bias_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
+ &input_allocation,
+ &bias_allocation,
+ &mut output_allocation,
input.num_cols,
input.length,
input.scale,
@@ -57,7 +60,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- output_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&output_allocation)
}
fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rs
index 6c99b5722..69b25307d 100644
--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rs
@@ -1,13 +1,16 @@
-use std::{fmt::Debug, ops::DerefMut};
+use std::fmt::Debug;
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
- ArrayContextExt, ArrayElement,
+ ArrayElement,
backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddSwapKernel},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::uzu_test;
+use crate::{
+ common::helpers::{alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
skip_buffer: Box<[T]>,
@@ -44,19 +47,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
.expect("Failed to create TensorAddSwapKernel");
let size = input.length as usize;
- let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");
- let main_array = context.create_array_from(&[size], &input.main_buffer, "");
+ let mut skip_allocation = alloc_allocation_with_data::<B, T>(&context, &input.skip_buffer[..size]);
+ let mut main_allocation = alloc_allocation_with_data::<B, T>(&context, &input.main_buffer[..size]);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- skip_array.buffer().borrow_mut().deref_mut(),
- main_array.buffer().borrow_mut().deref_mut(),
- input.length,
- &mut encoder,
- );
+ kernel.encode(&mut skip_allocation, &mut main_allocation, input.length, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- (skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())
+ (
+ allocation_to_vec::<B, T>(&skip_allocation),
+ allocation_to_vec::<B, T>(&main_allocation),
+ )
}
fn test<T: ArrayElement + Float + Debug>() {
diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rs
index 4adc00398..a3653860d 100644
--- a/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rs
@@ -1,16 +1,16 @@
-use std::{
- fmt::Debug,
- ops::{Deref, DerefMut},
-};
+use std::fmt::Debug;
+use half::{bf16, f16};
+use num_traits::Float;
use backend_uzu::{
- ArrayContextExt, ArrayElement,
+ ArrayElement,
backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorCopyKernel},
};
-use half::{bf16, f16};
-use num_traits::Float;
-use crate::uzu_test;
+use crate::{
+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
+};
struct Input<T: ArrayElement + Float> {
src: Box<[T]>,
@@ -43,19 +43,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
.expect("Failed to create TensorCopyKernel");
let size = input.length as usize;
- let src_array = context.create_array_from(&[size], &input.src, "");
- let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");
+ let src_allocation = alloc_allocation_with_data::<B, T>(&context, &input.src[..size]);
+ let mut dst_allocation = alloc_allocation::<B, T>(&context, size);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- src_array.buffer().borrow().deref(),
- dst_array.buffer().borrow_mut().deref_mut(),
- input.length,
- &mut encoder,
- );
+ kernel.encode(&src_allocation, &mut dst_allocation, input.length, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- dst_array.as_slice().to_vec()
+ allocation_to_vec::<B, T>(&dst_allocation)
}
fn test<T: ArrayElement + Float + Debug>() {
diff --git a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs
index 565d45d53..6e02a485c 100644
--- a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs
@@ -1,14 +1,14 @@
-use std::ops::{Deref, DerefMut};
-
use backend_uzu::{
- ArrayContextExt, ArrayElement,
backends::common::{
Backend, Context, Encoder, Kernels,
kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},
},
};
-use crate::uzu_test;
+use crate::{
+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ uzu_test,
+};
fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {
let context = B::Context::new().expect("Failed to create Context");
@@ -16,14 +16,14 @@ fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {
let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
.expect("Failed to create TokenCopySampledKernel");
- let src_array = context.create_array_from(&[1], &[src_value], "");
- let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");
+ let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);
+ let mut dst_allocation = alloc_allocation::<B, u64>(&context, 1);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);
+ kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let output = dst_array.as_slice::<u64>().to_vec();
+ let output = allocation_to_vec::<B, u64>(&dst_allocation);
assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());
}
@@ -33,14 +33,14 @@ fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {
let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)
.expect("Failed to create TokenCopyToResultsKernel");
- let src_array = context.create_array_from(&[1], &[src_value], "");
- let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");
+ let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);
+ let mut dst_allocation = alloc_allocation::<B, u32>(&context, 1);
let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);
+ kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let output = dst_array.as_slice::<u32>().to_vec();
+ let output = allocation_to_vec::<B, u32>(&dst_allocation);
assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());
}