A high-performance inference engine for AI models
Migration to allocations (#349)
LuckyIYI40708e01a9e4
diff --git a/crates/backend-uzu/src/array/array.rs b/crates/backend-uzu/src/array/array.rsindex 77d169e1f..bdfe6a44f 100644--- a/crates/backend-uzu/src/array/array.rs+++ b/crates/backend-uzu/src/array/array.rs@@ -1,60 +1,21 @@-use std::{cell::RefCell, rc::Rc};+use std::fmt;use crate::{DataType,array::size_for_shape,- backends::common::{AsBufferRangeMut, Backend, Buffer},+ backends::common::{+ Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, BufferRangeMut, BufferRangeRef,+ },};-#[derive(Debug)]-pub struct Array<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>> = <B as Backend>::DenseBuffer> {- pub(super) buffer: Rc<RefCell<Buf>>,+pub struct Array<B: Backend, BufferRange: AsBufferRangeMut<Buffer: Buffer<Backend = B>> = Allocation<B>> {+ pub(super) buffer_range: BufferRange,pub(super) offset: usize,pub(super) shape: Box<[usize]>,pub(super) data_type: DataType,}-impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, Buf> {- // Constructors- pub unsafe fn from_parts(- buffer: Rc<RefCell<Buf>>,- offset: usize,- shape: &[usize],- data_type: DataType,- ) -> Self {- let required_bytes = size_for_shape(shape, data_type);- let buffer_range = buffer.borrow().as_buffer_range_ref().range();- let buffer_range_size = buffer_range.end - buffer_range.start;-- assert!(- offset + required_bytes <= buffer_range_size,- "Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",- shape,- data_type,- offset,- offset + required_bytes,- buffer_range_size- );- Self {- buffer: buffer.clone(),- offset,- shape: shape.into(),- data_type,- }- }-- pub fn view(- &self,- shape: &[usize],- ) -> Self {- unsafe { Self::from_parts(self.buffer.clone(), self.offset, shape, self.data_type) }- }-- // Getters- pub fn buffer(&self) -> Rc<RefCell<Buf>> {- self.buffer.clone()- }-+impl<B: Backend, BufferRange: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, BufferRange> {pub fn offset(&self) -> usize {self.offset}@@ -74,15 +35,65 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Array<B, Bupub fn num_elements(&self) -> usize {self.shape.iter().product()}++ pub fn as_buffer_range_ref<'a>(&'a self) -> BufferRangeRef<'a, BufferRange::Buffer> {+ self.buffer_range.as_buffer_range_ref().subrange(self.offset..self.offset + self.size())+ }++ pub fn as_buffer_range_mut<'a>(&'a mut self) -> BufferRangeMut<'a, BufferRange::Buffer> {+ let offset = self.offset;+ let size = self.size();+ self.buffer_range.as_buffer_range_mut().subrange(offset..offset + size)+ }}-impl<B: Backend, Buf: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> Clone for Array<B, Buf> {- fn clone(&self) -> Self {+impl<B: Backend> Array<B, Allocation<B>> {+ pub unsafe fn from_allocation(+ allocation: Allocation<B>,+ offset: usize,+ shape: &[usize],+ data_type: DataType,+ ) -> Self {+ let required_bytes = size_for_shape(shape, data_type);+ let allocation_size = allocation.as_buffer_range_ref().range().len();+ assert!(offset + required_bytes <= allocation_size, "Array shape exceeds allocation length",);Self {- buffer: self.buffer.clone(),- offset: self.offset,- shape: self.shape.clone(),- data_type: self.data_type,+ buffer_range: allocation,+ offset,+ shape: shape.into(),+ data_type,}}++ pub fn allocation(&self) -> &Allocation<B> {+ &self.buffer_range+ }++ pub fn allocation_mut(&mut self) -> &mut Allocation<B> {+ &mut self.buffer_range+ }++ pub fn into_allocation(self) -> Allocation<B> {+ assert_eq!(self.offset, 0, "Array view cannot be converted into Allocation");+ assert_eq!(+ size_for_shape(&self.shape, self.data_type),+ self.buffer_range.as_buffer_range_ref().range().len(),+ "Partial Array view cannot be converted into Allocation",+ );+ self.buffer_range+ }+}++impl<B: Backend, BufferRange: AsBufferRangeMut<Buffer: Buffer<Backend = B>>> fmt::Debug for Array<B, BufferRange> {+ fn fmt(+ &self,+ formatter: &mut fmt::Formatter<'_>,+ ) -> fmt::Result {+ formatter+ .debug_struct("Array")+ .field("offset", &self.offset)+ .field("shape", &self.shape)+ .field("data_type", &self.data_type)+ .finish()+ }}diff --git a/crates/backend-uzu/src/array/context_ext.rs b/crates/backend-uzu/src/array/context_ext.rsindex 3b80227f0..7fa543aaf 100644--- a/crates/backend-uzu/src/array/context_ext.rs+++ b/crates/backend-uzu/src/array/context_ext.rs@@ -1,9 +1,7 @@-use std::{cell::RefCell, rc::Rc};-use crate::{Array, ArrayElement, DataType,array::size_for_shape,- backends::common::{Backend, Buffer, Context},+ backends::common::{AllocationType, Context},};pub trait ArrayContextExt: Context {@@ -11,23 +9,18 @@ pub trait ArrayContextExt: Context {&self,shape: &[usize],data_type: DataType,- label: &str,- ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {- let buffer_size_bytes = size_for_shape(shape, data_type);-- let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");- buffer.set_label(Some(label));-- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }+ ) -> Array<Self::Backend> {+ let size = size_for_shape(shape, data_type);+ let allocation = self.create_allocation(size, AllocationType::Global).expect("Failed to create allocation");+ unsafe { Array::from_allocation(allocation, 0, shape, data_type) }}fn create_array_zeros(&self,shape: &[usize],data_type: DataType,- label: &str,- ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {- let mut array = self.create_array_uninitialized(shape, data_type, label);+ ) -> Array<Self::Backend> {+ let mut array = self.create_array_uninitialized(shape, data_type);array.as_bytes_mut().fill(0);array}@@ -36,8 +29,7 @@ pub trait ArrayContextExt: Context {&self,shape: &[usize],data: &[T],- label: &str,- ) -> Array<Self::Backend, <Self::Backend as Backend>::DenseBuffer> {+ ) -> Array<Self::Backend> {let size_from_shape: usize = shape.iter().product();assert_eq!(data.len(),@@ -47,7 +39,7 @@ pub trait ArrayContextExt: Context {data.len());- let mut array = self.create_array_uninitialized(shape, T::data_type(), label);+ let mut array = self.create_array_uninitialized(shape, T::data_type());array.as_slice_mut().copy_from_slice(data);array}diff --git a/crates/backend-uzu/src/array/dense.rs b/crates/backend-uzu/src/array/dense.rsindex 1deda3906..45ac031d2 100644--- a/crates/backend-uzu/src/array/dense.rs+++ b/crates/backend-uzu/src/array/dense.rs@@ -3,18 +3,15 @@ use std::{ops::Range, os::raw::c_void, ptr::NonNull};use ndarray::{ArrayView, Dimension, IxDyn};use crate::{- ArrayElement,- array::Array,+ Array, ArrayElement,backends::common::{AsBufferRangeMut, Backend, DenseBuffer},};-impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<B, Buf> {+impl<B: Backend, BufferRange: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<B, BufferRange> {pub fn cpu_ptr(&self) -> NonNull<c_void> {- let buffer = self.buffer.borrow();- let buffer_range = buffer.as_buffer_range_ref();+ let buffer_range = self.as_buffer_range_ref();let range = buffer_range.range();-- unsafe { buffer_range.buffer().cpu_ptr().add(range.start + self.offset) }+ unsafe { buffer_range.buffer().cpu_ptr().add(range.start) }}pub fn as_bytes(&self) -> &[u8] {@@ -22,7 +19,12 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<}pub fn as_bytes_mut(&mut self) -> &mut [u8] {- unsafe { std::slice::from_raw_parts_mut(self.cpu_ptr().as_ptr() as *mut u8, self.size()) }+ let size = self.size();+ let buffer_range = self.as_buffer_range_mut();+ let range = buffer_range.range();+ unsafe {+ std::slice::from_raw_parts_mut((buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start), size)+ }}pub fn as_slice<T: ArrayElement>(&self) -> &[T] {@@ -41,66 +43,60 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<pub fn copy_from_array<OtherBackend: Backend,- OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,+ OtherBufferRange: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,>(&mut self,- other: &Array<OtherBackend, OtherBuf>,+ other: &Array<OtherBackend, OtherBufferRange>,) {- assert_eq!(self.shape, other.shape);- assert_eq!(self.data_type, other.data_type);+ assert_eq!(self.shape(), other.shape());+ assert_eq!(self.data_type(), other.data_type());self.as_bytes_mut().copy_from_slice(other.as_bytes());}pub fn copy_slice<OtherBackend: Backend,- OtherBuf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,+ OtherBufferRange: AsBufferRangeMut<Buffer: DenseBuffer<Backend = OtherBackend>>,>(&mut self,- source: &Array<OtherBackend, OtherBuf>,+ source: &Array<OtherBackend, OtherBufferRange>,axis: usize,src_range: Range<usize>,dst_offset: usize,) {assert_eq!(self.shape().len(), source.shape().len(), "Rank mismatch");assert!(axis < self.shape().len(), "Axis out of bounds");- for (i, (a, b)) in self.shape().iter().zip(source.shape()).enumerate() {- if i != axis {- assert_eq!(a, b, "Shapes must match on all non-sliced axes");+ for (dimension_index, (destination_dim, source_dim)) in self.shape().iter().zip(source.shape()).enumerate() {+ if dimension_index != axis {+ assert_eq!(destination_dim, source_dim, "Shapes must match on all non-sliced axes");}}assert_eq!(self.data_type(), source.data_type(), "Arrays must have the same data type");- let elem_size = self.data_type().size_in_bytes();-- // The number of contiguous elements to copy for each slice operation- let block_size_elems = self.shape().iter().skip(axis + 1).product::<usize>();- let block_size_bytes = block_size_elems * elem_size;-- // The total number of blocks to copy+ let element_size = self.data_type().size_in_bytes();+ let block_size_elements = self.shape().iter().skip(axis + 1).product::<usize>();+ let block_size_bytes = block_size_elements * element_size;let num_blocks: usize = self.shape().iter().take(axis).product();-- // Strides between the start of each block- let src_stride_bytes = source.shape()[axis] * block_size_bytes;- let dst_stride_bytes = self.shape()[axis] * block_size_bytes;+ let source_stride_bytes = source.shape()[axis] * block_size_bytes;+ let destination_stride_bytes = self.shape()[axis] * block_size_bytes;let rows_to_copy = src_range.end - src_range.start;assert!(dst_offset + rows_to_copy <= self.shape()[axis]);assert!(src_range.end <= source.shape()[axis]);- let src_buf = source.as_bytes();- let dst_buf = self.as_bytes_mut();+ let source_bytes = source.as_bytes();+ let destination_bytes = self.as_bytes_mut();let copy_bytes = rows_to_copy * block_size_bytes;- for i in 0..num_blocks {- let src_block_start = i * src_stride_bytes;- let dst_block_start = i * dst_stride_bytes;+ for block_index in 0..num_blocks {+ let source_block_start = block_index * source_stride_bytes;+ let destination_block_start = block_index * destination_stride_bytes;- let src_start = src_block_start + src_range.start * block_size_bytes;- let dst_start = dst_block_start + dst_offset * block_size_bytes;+ let source_start = source_block_start + src_range.start * block_size_bytes;+ let destination_start = destination_block_start + dst_offset * block_size_bytes;- let src_slice = &src_buf[src_start..src_start + copy_bytes];- let dst_slice = &mut dst_buf[dst_start..dst_start + copy_bytes];- dst_slice.copy_from_slice(src_slice);+ let source_slice = &source_bytes[source_start..source_start + copy_bytes];+ let destination_slice = &mut destination_bytes[destination_start..destination_start + copy_bytes];+ destination_slice.copy_from_slice(source_slice);}}@@ -110,15 +106,15 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<) {assert_eq!(self.data_type(), T::data_type());- let dst_slice = self.as_slice_mut::<T>();+ let destination_slice = self.as_slice_mut::<T>();- if let Some(src_slice) = view.as_slice_memory_order() {- assert!(src_slice.len() <= dst_slice.len());- dst_slice[..src_slice.len()].copy_from_slice(src_slice);+ if let Some(source_slice) = view.as_slice_memory_order() {+ assert!(source_slice.len() <= destination_slice.len());+ destination_slice[..source_slice.len()].copy_from_slice(source_slice);} else {- assert!(view.len() <= dst_slice.len());- for (d, s) in dst_slice.iter_mut().zip(view.iter()) {- *d = *s;+ assert!(view.len() <= destination_slice.len());+ for (destination, source) in destination_slice.iter_mut().zip(view.iter()) {+ *destination = *source;}}}@@ -126,10 +122,10 @@ impl<B: Backend, Buf: AsBufferRangeMut<Buffer: DenseBuffer<Backend = B>>> Array<fn validate_element_type<T: ArrayElement>(&self) {assert_eq!(T::data_type(),- self.data_type,+ self.data_type(),"Invalid data type, expected {:?}, actual {:?}",T::data_type(),- self.data_type+ self.data_type());}}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime.rs b/crates/backend-uzu/src/audio/nanocodec/runtime.rsindex 48b657f4e..54ee95c0e 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime.rs@@ -1,5 +1,5 @@use std::{- cell::{Cell, RefCell},+ cell::RefCell,collections::{BTreeMap, HashMap},fs::File,path::Path,@@ -9,8 +9,8 @@ use std::{use serde::Deserialize;use crate::{- DataType,- array::{Array, ArrayContextExt, size_for_shape},+ DataType, allocation_to_vec,+ array::{ArrayContextExt, size_for_shape},audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid},backends::common::{Backend, Context, Encoder, Kernels, Pending,@@ -26,11 +26,7 @@ use crate::{EmbeddingConfigCommon, InnerModelConfig, TtsAudioDecoderConfig, TtsConfig,},encodable_block::{Decoder, EncodingParameters, LayerExecutables, RMSNorm},- forward_pass::{- model_shape::ModelShape,- scratch_buffers::ScratchBuffers,- state::{ArrayId, ForwardPassState, SharedBuffers},- },+ forward_pass::{model_shape::ModelShape, state::SharedBuffers},parameters::ParameterLoader,};@@ -324,22 +320,14 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {let kernel = <B::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32).map_err(|err| AudioError::Runtime(format!("failed to initialize fsq decode kernel: {err}")))?;- let mut tokens_array = context.create_array_zeros(- &[batch_size, self.config.num_groups(), frames],- DataType::I32,- "nanocodec_fsq_decode_tokens",- );- tokens_array.as_slice_mut::<i32>().copy_from_slice(&tokens_i32);+ let tokens_allocation =+ context.create_array_from(&[batch_size, self.config.num_groups(), frames], &tokens_i32).into_allocation();- let mut lengths_array =- context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");- lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);+ let lengths_allocation = context.create_array_from(&[batch_size], &lengths_i32).into_allocation();- let output = context.create_array_zeros(- &[batch_size, self.config.channels(), frames],- DataType::F32,- "nanocodec_fsq_decode_output",- );+ let mut output = context+ .create_array_uninitialized(&[batch_size, self.config.channels(), frames], DataType::F32)+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;@@ -348,27 +336,18 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {let frames_i32 = usize_to_i32(frames, "frames")?;let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;- {- let tokens_buffer = tokens_array.buffer();- let tokens_buffer = tokens_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();-- kernel.encode(- &*tokens_buffer,- &mut *output_buffer,- &*lengths_buffer,- num_groups_i32,- frames_i32,- codebook_dim_i32,- self.config.num_levels_per_group(),- self.config.dim_base_index(),- batch_size_i32,- &mut encoder,- );- }+ kernel.encode(+ &tokens_allocation,+ &mut output,+ &lengths_allocation,+ num_groups_i32,+ frames_i32,+ codebook_dim_i32,+ self.config.num_levels_per_group(),+ self.config.dim_base_index(),+ batch_size_i32,+ &mut encoder,+ );encoder.end_encoding()@@ -377,7 +356,7 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {.map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ decode command buffer: {err}")))?;Ok(DecodedPaddedAudio {- samples: output.as_slice::<f32>().to_vec(),+ samples: allocation_to_vec::<B, f32>(&output),channels: self.config.channels(),frames,lengths: lengths_usize,@@ -415,21 +394,14 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {let kernel = <B::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32).map_err(|err| AudioError::Runtime(format!("failed to initialize fsq encode kernel: {err}")))?;- let mut input = context.create_array_zeros(- &[batch_size, self.config.channels(), frames],- DataType::F32,- "nanocodec_fsq_encode_input",- );- input.as_slice_mut::<f32>().copy_from_slice(&padded_input);+ let input =+ context.create_array_from(&[batch_size, self.config.channels(), frames], &padded_input).into_allocation();- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");- lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);+ let lengths = context.create_array_from(&[batch_size], &lengths_i32).into_allocation();- let tokens = context.create_array_zeros(- &[batch_size, self.config.num_groups(), frames],- DataType::I32,- "nanocodec_fsq_encode_tokens",- );+ let mut tokens = context+ .create_array_uninitialized(&[batch_size, self.config.num_groups(), frames], DataType::I32)+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;@@ -438,28 +410,19 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {let frames_i32 = usize_to_i32(frames, "frames")?;let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;- {- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let tokens_buffer = tokens.buffer();- let mut tokens_buffer = tokens_buffer.borrow_mut();- let lengths_buffer = lengths.buffer();- let lengths_buffer = lengths_buffer.borrow();-- kernel.encode(- &*input_buffer,- &mut *tokens_buffer,- &*lengths_buffer,- num_groups_i32,- frames_i32,- codebook_dim_i32,- self.config.num_levels_per_group(),- self.config.dim_base_index(),- self.config.eps(),- batch_size_i32,- &mut encoder,- );- }+ kernel.encode(+ &input,+ &mut tokens,+ &lengths,+ num_groups_i32,+ frames_i32,+ codebook_dim_i32,+ self.config.num_levels_per_group(),+ self.config.dim_base_index(),+ self.config.eps(),+ batch_size_i32,+ &mut encoder,+ );encoder.end_encoding()@@ -467,8 +430,9 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {.wait_until_completed().map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ encode command buffer: {err}")))?;- let mut tokens_u32 = vec![0_u32; tokens.num_elements()];- for (index, &token) in tokens.as_slice::<i32>().iter().enumerate() {+ let encoded_tokens = allocation_to_vec::<B, i32>(&tokens);+ let mut tokens_u32 = vec![0_u32; encoded_tokens.len()];+ for (index, &token) in encoded_tokens.iter().enumerate() {if token < 0 {return Err(AudioError::Runtime(format!("fsq encode returned negative token at index {index}: {token}"@@ -539,8 +503,9 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {pub fn end_decode_stream(&self,- _state: AudioDecodeStreamState,+ state: AudioDecodeStreamState,) -> AudioResult<()> {+ drop(state);Ok(())}@@ -766,11 +731,6 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {codebooks,window_frames,)?;- // The scratch buffers inside the workspace were encoded into the- // submitted command buffer. Reset the workspace so the next decode- // pass allocates fresh buffers instead of reusing ones the in-flight- // command buffer may still be reading.- resources.reset_for_pending();let audio_offset_frames = window_start.checked_mul(fishaudio.upsample_factor).ok_or(AudioError::Runtime("stream audio offset overflow".to_string()))?;diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rsindex 8c19f5df3..6970aa9cf 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/profile.rs@@ -1,44 +1,76 @@use std::sync::mpsc::Receiver;use super::*;+use crate::{allocation_to_vec, array::Array, backends::common::Allocation};-pub struct SubmittedDecodedPaddedAudio<B: Backend> {- pub(in crate::audio::nanocodec::runtime) output: Array<B>,- pub(in crate::audio::nanocodec::runtime) channels: usize,- pub(in crate::audio::nanocodec::runtime) frames: usize,- pub(in crate::audio::nanocodec::runtime) lengths: Vec<usize>,- pub(in crate::audio::nanocodec::runtime) final_command_buffer: Option<Pending<B>>,- pub(in crate::audio::nanocodec::runtime) completion_notification: Option<Receiver<()>>,+pub enum SubmittedDecodedPaddedAudio<B: Backend> {+ Ready(DecodedPaddedAudio),+ Pending {+ output: Allocation<B>,+ retained_inputs: Box<[Array<B>]>,+ data_type: DataType,+ channels: usize,+ frames: usize,+ lengths: Vec<usize>,+ final_command_buffer: Pending<B>,+ completion_notification: Receiver<()>,+ },}impl<B: Backend> SubmittedDecodedPaddedAudio<B> {pub(crate) fn is_complete(&self) -> bool {use std::sync::mpsc::TryRecvError;- self.completion_notification.as_ref().is_none_or(|notification| match notification.try_recv() {- Ok(()) | Err(TryRecvError::Disconnected) => true,- Err(TryRecvError::Empty) => false,- })+ match self {+ Self::Ready(_) => true,+ Self::Pending {+ completion_notification,+ ..+ } => match completion_notification.try_recv() {+ Ok(()) | Err(TryRecvError::Disconnected) => true,+ Err(TryRecvError::Empty) => false,+ },+ }}- pub(in crate::audio::nanocodec::runtime) fn resolve(mut self) -> AudioResult<DecodedPaddedAudio> {- if let Some(command_buffer) = self.final_command_buffer.take() {- command_buffer.wait_until_completed().map_err(|err| {- AudioError::Runtime(format!("failed to wait for FishAudio decoder command buffer: {err}"))- })?;+ pub(in crate::audio::nanocodec::runtime) fn resolve(self) -> AudioResult<DecodedPaddedAudio> {+ match self {+ Self::Ready(decoded) => Ok(decoded),+ Self::Pending {+ output,+ retained_inputs,+ data_type,+ channels,+ frames,+ lengths,+ final_command_buffer,+ completion_notification: _,+ } => {+ let completed_command_buffer = final_command_buffer.wait_until_completed().map_err(|err| {+ AudioError::Runtime(format!("failed to wait for FishAudio decoder command buffer: {err}"))+ })?;+ drop(retained_inputs);+ let samples_result: AudioResult<Vec<f32>> = match data_type {+ DataType::F32 => Ok(allocation_to_vec::<B, f32>(&output)),+ DataType::F16 => {+ Ok(allocation_to_vec::<B, half::f16>(&output).iter().map(|&value| f32::from(value)).collect())+ },+ DataType::BF16 => {+ Ok(allocation_to_vec::<B, half::bf16>(&output).iter().map(|&value| f32::from(value)).collect())+ },+ dt => Err(AudioError::Runtime(format!("unsupported vocoder output dtype: {dt:?}"))),+ };+ drop(output);+ drop(completed_command_buffer);+ let samples = samples_result?;+ Ok(DecodedPaddedAudio {+ samples,+ channels,+ frames,+ lengths,+ })+ },}- let samples: Vec<f32> = match self.output.data_type() {- DataType::F32 => self.output.as_slice::<f32>().to_vec(),- DataType::F16 => self.output.as_slice::<half::f16>().iter().map(|&v| f32::from(v)).collect(),- DataType::BF16 => self.output.as_slice::<half::bf16>().iter().map(|&v| f32::from(v)).collect(),- dt => return Err(AudioError::Runtime(format!("unsupported vocoder output dtype: {dt:?}"))),- };- Ok(DecodedPaddedAudio {- samples,- channels: self.channels,- frames: self.frames,- lengths: self.lengths,- })}}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rsindex 3e2d90914..cc0b16259 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs@@ -1,9 +1,10 @@use super::*;+use crate::array::{Array, ArrayContextExt};pub(super) type BackendParameterTree<'loader, B> = crate::parameters::ParameterTree<'loader, <B as Backend>::Context>;pub(super) fn read_float_array<B: Backend, const RANK: usize>(- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,name: &str,expected_data_type: DataType,) -> AudioResult<([usize; RANK], Array<B>)> {@@ -16,8 +17,8 @@ pub(super) fn read_float_array<B: Backend, const RANK: usize>(}if array.shape().len() != RANK {return Err(AudioError::Runtime(format!(- "expected rank-{RANK} tensor for '{name}', got shape {:?}",- array.shape()+ "expected rank-{RANK} tensor for '{name}', got rank {}",+ array.shape().len())));}let mut dims = [0usize; RANK];@@ -26,7 +27,7 @@ pub(super) fn read_float_array<B: Backend, const RANK: usize>(}pub(super) fn read_float_vector_exact<B: Backend>(- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,name: &str,expected_len: usize,expected_data_type: DataType,@@ -42,7 +43,7 @@ pub(super) fn read_float_vector_exact<B: Backend>(}pub(super) fn read_float_matrix_exact<B: Backend>(- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,name: &str,expected_rows: usize,expected_cols: usize,@@ -58,21 +59,29 @@ pub(super) fn read_float_matrix_exact<B: Backend>(Ok(array)}-pub(super) fn outer_axis_view<B: Backend>(- array: &Array<B>,+pub(super) fn copy_to_outer_axis_slice<B: Backend>(+ destination: &mut Array<B>,index: usize,+ source: &Array<B>,slice_shape: &[usize],-) -> AudioResult<Array<B>> {- let slice_bytes = size_for_shape(slice_shape, array.data_type());- let offset = index- .checked_mul(slice_bytes)- .and_then(|value| value.checked_add(array.offset()))- .ok_or(AudioError::Runtime("array outer-axis view offset overflow".to_string()))?;- Ok(unsafe { Array::from_parts(array.buffer(), offset, slice_shape, array.data_type()) })+) -> AudioResult<()> {+ let slice_bytes = size_for_shape(slice_shape, source.data_type());+ let offset =+ index.checked_mul(slice_bytes).ok_or(AudioError::Runtime("outer-axis slice offset overflow".to_string()))?;+ let end =+ offset.checked_add(slice_bytes).ok_or(AudioError::Runtime("outer-axis slice end overflow".to_string()))?;+ if end > destination.size() {+ return Err(AudioError::Runtime(format!(+ "outer-axis slice exceeds destination: end {end}, destination size {}",+ destination.size()+ )));+ }+ destination.as_bytes_mut()[offset..end].copy_from_slice(source.as_bytes());+ Ok(())}pub(super) fn read_conv1d_layer<B: Backend>(- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,data_type: DataType,dilation: usize,groups: usize,@@ -100,7 +109,7 @@ pub(super) fn read_conv1d_layer<B: Backend>(}pub(super) fn read_conv_transpose1d_layer<B: Backend>(- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,data_type: DataType,stride: usize,groups: usize,@@ -139,7 +148,7 @@ pub(super) fn read_conv_transpose1d_layer<B: Backend>(}pub(super) fn read_pointwise_conv_layer<B: Backend>(- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,data_type: DataType,expected_out_dim: usize,expected_in_dim: usize,@@ -147,7 +156,7 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(let weight = read_float_matrix_exact::<B>(tree, "weights", expected_out_dim, expected_in_dim, data_type)?;let bias = read_float_vector_exact::<B>(tree, "biases", expected_out_dim, data_type)?;Ok(StructuredAudioPointwiseConv {- weight: weight.view(&[expected_out_dim, expected_in_dim, 1]),+ weight,bias,cin: expected_in_dim,cout: expected_out_dim,@@ -156,19 +165,18 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(pub(super) fn read_norm_layer<B: Backend>(context: &Rc<B::Context>,- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,data_type: DataType,channels: usize,epsilon: f32,subtract_mean: bool,use_bias: bool,- label_prefix: &str,) -> AudioResult<StructuredAudioNorm<B>> {let scales = read_float_vector_exact::<B>(tree, "scales", channels, data_type)?;let bias = if use_bias {read_float_vector_exact::<B>(tree, "biases", channels, data_type)?} else {- context.create_array_zeros(&[channels], data_type, &format!("{label_prefix}_bias"))+ context.create_array_zeros(&[channels], data_type)};Ok(StructuredAudioNorm {scales,@@ -180,7 +188,7 @@ pub(super) fn read_norm_layer<B: Backend>(pub(super) fn read_convnext_layer<B: Backend>(context: &Rc<B::Context>,- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,data_type: DataType,norm_config: &DescriptAudioConvNeXtNormConfig,) -> AudioResult<StructuredAudioConvNeXt<B>> {@@ -201,13 +209,6 @@ pub(super) fn read_convnext_layer<B: Backend>(dilation: 1,groups: depthwise_shape[0],};- if depthwise_conv.bias.shape()[0] != depthwise_conv.cout {- return Err(AudioError::Runtime(format!(- "ConvNeXt depthwise bias mismatch at 'dwconv.biases': expected {}, got {}",- depthwise_conv.cout,- depthwise_conv.bias.shape()[0]- )));- }if depthwise_conv.cout == 0 || depthwise_conv.kernel_size == 0 {return Err(AudioError::InvalidTokenCardinality);}@@ -226,7 +227,6 @@ pub(super) fn read_convnext_layer<B: Backend>(norm_config.epsilon,norm_config.subtract_mean,norm_config.use_bias,- "structured_audio_convnext_norm",)?;let (pwconv1_shape, _) = read_float_array::<B, 2>(&tree.subtree("pwconv1")?, "weights", data_type)?;if pwconv1_shape[1] != channels {@@ -246,7 +246,7 @@ pub(super) fn read_convnext_layer<B: Backend>(}pub(super) fn read_residual_unit_layer<B: Backend>(- tree: &BackendParameterTree<'_, B>,+ tree: &BackendParameterTree<B>,data_type: DataType,dilation: usize,) -> AudioResult<StructuredAudioResidualUnit<B>> {@@ -270,13 +270,6 @@ pub(super) fn read_residual_unit_layer<B: Backend>(}alpha};- if snake1_alpha.shape()[0] != conv1.cin {- return Err(AudioError::Runtime(format!(- "residual snake1 alpha mismatch: expected {}, got {}",- conv1.cin,- snake1_alpha.shape()[0]- )));- }let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, 1, 1)?;Ok(StructuredAudioResidualUnit {snake1_alpha,@@ -288,7 +281,7 @@ pub(super) fn read_residual_unit_layer<B: Backend>(pub(super) fn build_vocoder_graph_from_tree<B: Backend>(context: &Rc<B::Context>,- root: &BackendParameterTree<'_, B>,+ root: &BackendParameterTree<B>,config: &DescriptAudioCodecConfig,data_type: DataType,) -> AudioResult<StructuredAudioDecoderGraph<B>> {@@ -330,11 +323,7 @@ pub(super) fn build_vocoder_graph_from_tree<B: Backend>(let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1)?;let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3)?;let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9)?;- if snake_alpha.shape()[0] != trans_conv.cin- || res_unit1.conv1.cin != channels- || res_unit2.conv1.cin != channels- || res_unit3.conv1.cin != channels- {+ if res_unit1.conv1.cin != channels || res_unit2.conv1.cin != channels || res_unit3.conv1.cin != channels {return Err(AudioError::Runtime(format!("structured audio decoder block {index} channel mismatch in exported weights")));diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rsindex 0c2ec5054..3d94a9f7f 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/context.rs@@ -1,4 +1,10 @@use super::*;+use crate::array::Array;++pub(super) struct QuantizerDecodeOutput<B: Backend> {+ pub(super) output: Array<B>,+ pub(super) retained_inputs: Box<[Array<B>]>,+}impl StructuredAudioCodecGraph {fn conv1d_input_context<B: Backend>(layer: &StructuredAudioConv1d<B>) -> AudioResult<usize> {@@ -97,14 +103,13 @@ impl StructuredAudioCodecGraph {pub(super) fn decode_quantizer_to_nsc_array_enqueued<B: Backend>(&self,resources: &StructuredAudioRuntimeResources<B>,- context: &Rc<B::Context>,encoder: &mut Encoder<B>,tokens: &[u32],lengths: &[usize],batch_size: usize,codebooks: usize,frames: usize,- ) -> AudioResult<Array<B>> {+ ) -> AudioResult<QuantizerDecodeOutput<B>> {if codebooks != self.total_codebooks {return Err(AudioError::Runtime(format!("structured audio codebook mismatch: expected {}, got {codebooks}",@@ -148,11 +153,24 @@ impl StructuredAudioCodecGraph {let mut lengths_array = resources.length_staging(batch_size);lengths_array.as_slice_mut::<i32>()[..lengths_i32.len()].copy_from_slice(&lengths_i32);- let output = context.create_array_zeros(- &[batch_size, frames, self.input_dim],- quantizer_resources.data_type,- "structured_audio_quantizer_output_nsc",- );+ let output = unsafe {+ Array::from_allocation(+ encoder+ .allocate_scratch(size_for_shape(+ &[batch_size, frames, self.input_dim],+ quantizer_resources.data_type,+ ))+ .map_err(|err| {+ AudioError::Runtime(format!("failed to allocate structured audio quantizer output: {err}"))+ })?,+ 0,+ &[batch_size, frames, self.input_dim],+ quantizer_resources.data_type,+ )+ };+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();let batch_i32 = usize_to_i32(batch_size, "batch_size")?;let codebooks_i32 = usize_to_i32(codebooks, "codebooks")?;let frames_i32 = usize_to_i32(frames, "frames")?;@@ -162,34 +180,16 @@ impl StructuredAudioCodecGraph {let semantic_cardinality_i32 = usize_to_i32(quantizer_resources.semantic_cardinality, "semantic_cardinality")?;let residual_cardinality_i32 = usize_to_i32(quantizer_resources.residual_cardinality, "residual_cardinality")?;- let tokens_buffer = tokens_array.buffer();- let tokens_buffer = tokens_buffer.borrow();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();- let semantic_codebook_buffer = quantizer_resources.semantic_codebook.buffer();- let semantic_codebook_buffer = semantic_codebook_buffer.borrow();- let semantic_out_proj_buffer = quantizer_resources.semantic_out_proj.buffer();- let semantic_out_proj_buffer = semantic_out_proj_buffer.borrow();- let semantic_out_bias_buffer = quantizer_resources.semantic_out_bias.buffer();- let semantic_out_bias_buffer = semantic_out_bias_buffer.borrow();- let residual_codebooks_buffer = quantizer_resources.residual_codebooks.buffer();- let residual_codebooks_buffer = residual_codebooks_buffer.borrow();- let residual_out_proj_buffer = quantizer_resources.residual_out_proj.buffer();- let residual_out_proj_buffer = residual_out_proj_buffer.borrow();- let residual_out_bias_buffer = quantizer_resources.residual_out_bias.buffer();- let residual_out_bias_buffer = residual_out_bias_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();quantizer_resources.kernel.encode(- &*tokens_buffer,- &*lengths_buffer,- &*semantic_codebook_buffer,- &*semantic_out_proj_buffer,- &*semantic_out_bias_buffer,- &*residual_codebooks_buffer,- &*residual_out_proj_buffer,- &*residual_out_bias_buffer,- &mut *output_buffer,+ tokens_array.allocation(),+ lengths_array.allocation(),+ quantizer_resources.semantic_codebook.allocation(),+ quantizer_resources.semantic_out_proj.allocation(),+ quantizer_resources.semantic_out_bias.allocation(),+ quantizer_resources.residual_codebooks.allocation(),+ quantizer_resources.residual_out_proj.allocation(),+ quantizer_resources.residual_out_bias.allocation(),+ &mut output,batch_i32,codebooks_i32,frames_i32,@@ -200,7 +200,11 @@ impl StructuredAudioCodecGraph {residual_cardinality_i32,encoder,);+ let retained_inputs = vec![tokens_array, lengths_array].into_boxed_slice();- Ok(output)+ Ok(QuantizerDecodeOutput {+ output: unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) },+ retained_inputs,+ })}}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rsindex c5b448575..08204c37c 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/decode.rs@@ -1,7 +1,7 @@use std::sync::mpsc::channel;use super::*;-use crate::audio::nanocodec::runtime::profile::SubmittedDecodedPaddedAudio;+use crate::{array::Array, audio::nanocodec::runtime::profile::SubmittedDecodedPaddedAudio};impl StructuredAudioCodecGraph {fn run_residual_unit_enqueued<B: Backend>(@@ -18,52 +18,56 @@ impl StructuredAudioCodecGraph {kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {let ws = &resources.decode_workspace;- let ctx = resources.context();- let data_type = input.data_type();-- let residual = input.clone();- let mut x = snake1d_enqueue(+ let res_snake1 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);+ let x = snake1d_enqueue(encoder,input,- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake1")?,+ res_snake1,&unit.snake1_alpha,batch_size,channels,seq_len,+ self.vocoder_data_type,kernels,)?;- x = causal_conv1d_grouped_enqueue(+ let res_conv1 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);+ let x = causal_conv1d_grouped_enqueue(encoder,&x,- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv1")?,+ res_conv1,&unit.conv1,SequenceLayout::Ncs,lengths,lengths_array,batch_size,seq_len,+ self.vocoder_data_type,kernels,)?;- x = snake1d_enqueue(+ let res_snake2 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);+ let x = snake1d_enqueue(encoder,&x,- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake2")?,+ res_snake2,&unit.snake2_alpha,batch_size,channels,seq_len,+ self.vocoder_data_type,kernels,)?;+ let res_conv2 = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);causal_conv1d_grouped_residual_enqueue(encoder,&x,- &residual,- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv2")?,+ input,+ res_conv2,&unit.conv2,lengths,lengths_array,batch_size,seq_len,+ self.vocoder_data_type,kernels,)}@@ -86,15 +90,12 @@ impl StructuredAudioCodecGraph {.ok_or(AudioError::Runtime("structured audio length scaling overflow".to_string()))}).collect::<AudioResult<Vec<_>>>()?;- let context = resources.context().clone();- return Ok(SubmittedDecodedPaddedAudio {- output: context.create_array_zeros(&[0], DataType::F32, "structured_audio_empty_decode_output"),+ return Ok(SubmittedDecodedPaddedAudio::Ready(DecodedPaddedAudio {+ samples: Vec::new(),channels: 1,frames: out_lengths.iter().copied().max().unwrap_or(0),lengths: out_lengths,- final_command_buffer: None,- completion_notification: None,- });+ }));}let mut lengths_i32 = lengths@@ -113,34 +114,10 @@ impl StructuredAudioCodecGraph {let vocoder_graph = self.vocoder_graph(resources)?;let kernels = resources.kernels(self.vocoder_data_type)?;- let worst_case_bytes = {- let scratch_bytes = |channels: usize, frame_count: usize| -> usize {- size_for_shape(&[batch_size, channels, frame_count], self.vocoder_data_type)- };- let mut max_bytes = 0usize;- let mut current_frames = frames;- for (trans_conv, convnext) in vocoder_graph.upsample_blocks.iter() {- current_frames = current_frames.saturating_mul(trans_conv.stride);- max_bytes = max_bytes.max(scratch_bytes(trans_conv.cout, current_frames));- max_bytes = max_bytes.max(scratch_bytes(convnext.pwconv1.cout, current_frames));- }- let mut current_channels = vocoder_graph.first_conv.cout;- max_bytes = max_bytes.max(scratch_bytes(current_channels, current_frames));- for block in vocoder_graph.decoder_blocks.iter() {- max_bytes = max_bytes.max(scratch_bytes(current_channels, current_frames));- current_frames = current_frames.saturating_mul(block.trans_conv.stride);- current_channels = block.trans_conv.cout;- max_bytes = max_bytes.max(scratch_bytes(current_channels, current_frames));- }- max_bytes- };- ws.prime_for(context.as_ref(), worst_case_bytes)?;-let mut x;let mut x_layout = SequenceLayout::Nsc;let quantized_nsc = self.decode_quantizer_to_nsc_array_enqueued(resources,- &context,&mut encoder,tokens,lengths,@@ -148,12 +125,11 @@ impl StructuredAudioCodecGraph {codebooks,frames,)?;- let _ = codebooks;+ let retained_inputs = quantized_nsc.retained_inputs;x = self.apply_post_module_enqueued(resources,- &context,&mut encoder,- &quantized_nsc,+ quantized_nsc.output,lengths,batch_size,frames,@@ -162,11 +138,9 @@ impl StructuredAudioCodecGraph {let mut current_channels = self.input_dim;let mut current_frames = frames;let mut next_lengths_i32 = vec![0_i32; lengths_i32.len()];- let mut lengths_array = ws.lengths_array(&context, 0, lengths_i32.len());+ let mut lengths_array = ws.lengths_array(&mut encoder, lengths_i32.len());lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);- let mut next_lengths_array = ws.lengths_array(&context, 1, lengths_i32.len());-- let data_type = x.data_type();+ let mut next_lengths_array = ws.lengths_array(&mut encoder, lengths_i32.len());for (block_index, (trans_conv, convnext)) in vocoder_graph.upsample_blocks.iter().enumerate() {if trans_conv.cin != current_channels {@@ -181,10 +155,12 @@ impl StructuredAudioCodecGraph {scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, trans_conv.stride)?;next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);+ let up_tconv =+ ws.next_scratch(&mut encoder, &[batch_size, trans_conv.cout, next_frames], self.vocoder_data_type);x = causal_conv_transpose1d_causal_pad_enqueue(&mut encoder,&x,- ws.next_scratch(&context, &[batch_size, trans_conv.cout, next_frames], data_type, "up_tconv")?,+ up_tconv,trans_conv,&next_lengths_i32,batch_size,@@ -192,12 +168,13 @@ impl StructuredAudioCodecGraph {next_frames,x_layout,&next_lengths_array,+ self.vocoder_data_type,&kernels,).map_err(|err| {AudioError::Runtime(format!("structured audio upsample block {block_index} transpose_conv failed: {err} (x_len={}, batch_size={}, cin={}, seq_len_in={}, seq_len_out={})",- x.num_elements(),+ x.size(),batch_size,trans_conv.cin,current_frames,@@ -234,21 +211,22 @@ impl StructuredAudioCodecGraph {vocoder_graph.first_conv.cin, current_channels)));}+ let dec_first_conv = ws.next_scratch(+ &mut encoder,+ &[batch_size, vocoder_graph.first_conv.cout, current_frames],+ self.vocoder_data_type,+ );x = causal_conv1d_grouped_enqueue(&mut encoder,&x,- ws.next_scratch(- &context,- &[batch_size, vocoder_graph.first_conv.cout, current_frames],- data_type,- "dec_first_conv",- )?,+ dec_first_conv,&vocoder_graph.first_conv,x_layout,&lengths_i32,&lengths_array,batch_size,current_frames,+ self.vocoder_data_type,&kernels,)?;current_channels = vocoder_graph.first_conv.cout;@@ -260,14 +238,17 @@ impl StructuredAudioCodecGraph {block.trans_conv.cin, current_channels)));}+ let dec_snake =+ ws.next_scratch(&mut encoder, &[batch_size, current_channels, current_frames], self.vocoder_data_type);x = snake1d_enqueue(&mut encoder,&x,- ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "dec_snake")?,+ dec_snake,&block.snake_alpha,batch_size,current_channels,current_frames,+ self.vocoder_data_type,&kernels,)?;@@ -277,10 +258,15 @@ impl StructuredAudioCodecGraph {scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, block.trans_conv.stride)?;next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);+ let dec_tconv = ws.next_scratch(+ &mut encoder,+ &[batch_size, block.trans_conv.cout, next_frames],+ self.vocoder_data_type,+ );x = causal_conv_transpose1d_causal_pad_enqueue(&mut encoder,&x,- ws.next_scratch(&context, &[batch_size, block.trans_conv.cout, next_frames], data_type, "dec_tconv")?,+ dec_tconv,&block.trans_conv,&next_lengths_i32,batch_size,@@ -288,6 +274,7 @@ impl StructuredAudioCodecGraph {next_frames,SequenceLayout::Ncs,&next_lengths_array,+ self.vocoder_data_type,&kernels,)?;@@ -334,41 +321,43 @@ impl StructuredAudioCodecGraph {)?;}+ let final_snake =+ ws.next_scratch(&mut encoder, &[batch_size, current_channels, current_frames], self.vocoder_data_type);x = snake1d_enqueue(&mut encoder,&x,- ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "final_snake")?,+ final_snake,&vocoder_graph.final_snake_alpha,batch_size,current_channels,current_frames,+ self.vocoder_data_type,&kernels,)?;+ let final_conv = ws.next_scratch(+ &mut encoder,+ &[batch_size, vocoder_graph.final_conv.cout, current_frames],+ self.vocoder_data_type,+ );x = causal_conv1d_grouped_enqueue(&mut encoder,&x,- ws.next_scratch(- &context,- &[batch_size, vocoder_graph.final_conv.cout, current_frames],- data_type,- "final_conv",- )?,+ final_conv,&vocoder_graph.final_conv,SequenceLayout::Ncs,&lengths_i32,&lengths_array,batch_size,current_frames,+ self.vocoder_data_type,&kernels,)?;- // The final output is returned and held by the caller, so allocate a- // fresh array instead of using a scratch buffer.- x = tanh_enqueue(+ let tanh_output = ws.next_scratch(&mut encoder,- &x,- context.create_array_zeros(x.shape(), data_type, "structured_audio_tanh_output"),- &kernels,- )?;+ &[batch_size, vocoder_graph.final_conv.cout, current_frames],+ self.vocoder_data_type,+ );+ x = tanh_enqueue(&mut encoder, &x, tanh_output, &kernels)?;let (completion_sender, completion_notification) = channel();encoder.add_completion_handler({@@ -376,7 +365,6 @@ impl StructuredAudioCodecGraph {let _ = completion_sender.send(());}});- let final_command_buffer = Some(encoder.end_encoding().submit());let out_lengths = lengths_i32.into_iter().map(|length| {@@ -385,13 +373,16 @@ impl StructuredAudioCodecGraph {})}).collect::<AudioResult<Vec<_>>>()?;- Ok(SubmittedDecodedPaddedAudio {- output: x,+ let final_command_buffer = encoder.end_encoding().submit();+ Ok(SubmittedDecodedPaddedAudio::Pending {+ output: x.into_allocation(),+ retained_inputs,+ data_type: self.vocoder_data_type,channels: vocoder_graph.final_conv.cout,frames: current_frames,lengths: out_lengths,final_command_buffer,- completion_notification: Some(completion_notification),+ completion_notification,})}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rsindex 5f0051723..827917577 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/enqueue.rs@@ -1,5 +1,5 @@use super::*;-use crate::backends::common::gpu_types::ActivationType;+use crate::{array::Array, backends::common::gpu_types::ActivationType};pub(super) fn snake1d_enqueue<B: Backend>(encoder: &mut Encoder<B>,@@ -9,44 +9,33 @@ pub(super) fn snake1d_enqueue<B: Backend>(batch_size: usize,channels: usize,seq_len: usize,+ data_type: DataType,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {- let expected_input = checked_product(&[batch_size, channels, seq_len])?;- if input.num_elements() != expected_input {+ let expected_input = size_for_shape(&[batch_size, channels, seq_len], data_type);+ if input.size() != expected_input {return Err(AudioError::InvalidTokenShape {expected_tokens: expected_input,- actual_tokens: input.num_elements(),+ actual_tokens: input.size(),});}- if alpha.shape() != [channels] {+ if alpha.size() != size_for_shape(&[channels], data_type) {return Err(AudioError::InvalidTokenShape {expected_tokens: channels,actual_tokens: alpha.num_elements(),});}- let data_type = input.data_type();- if alpha.data_type() != data_type {- return Err(AudioError::Runtime(format!(- "snake alpha dtype mismatch: expected {:?}, got {:?}",- data_type,- alpha.data_type()- )));- }-let channels_i32 = usize_to_i32(channels, "channels")?;let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;let batch_i32 = usize_to_i32(batch_size, "batch_size")?;- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let alpha_buffer = alpha.buffer();- let alpha_buffer = alpha_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();kernels.half_snake.encode(- &*input_buffer,- &*alpha_buffer,- &mut *output_buffer,+ input.allocation(),+ alpha.allocation(),+ &mut output,channels_i32,seq_len_i32,channels_i32,@@ -56,11 +45,12 @@ pub(super) fn snake1d_enqueue<B: Backend>(encoder,);- Ok(output)+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}fn validate_grouped_conv_params<B: Backend>(input: &Array<B>,+ data_type: DataType,layer: &StructuredAudioConv1d<B>,lengths: &[i32],batch_size: usize,@@ -76,31 +66,22 @@ fn validate_grouped_conv_params<B: Backend>(return Err(AudioError::InvalidTokenCardinality);}let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];- if layer.weight.shape() != expected_weight_shape {+ if layer.weight.size() != size_for_shape(&expected_weight_shape, data_type) {return Err(AudioError::Runtime(format!(- "causal conv1d weight shape mismatch: expected {:?}, got {:?}",- expected_weight_shape,- layer.weight.shape()+ "causal conv1d weight shape mismatch: expected {:?}",+ expected_weight_shape)));}- if layer.bias.shape() != [layer.cout] {- return Err(AudioError::Runtime(format!(- "causal conv1d bias shape mismatch: expected [{}], got {:?}",- layer.cout,- layer.bias.shape()- )));+ if layer.bias.size() != size_for_shape(&[layer.cout], data_type) {+ return Err(AudioError::Runtime(format!("causal conv1d bias shape mismatch: expected [{}]", layer.cout)));}- let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;- if input.num_elements() != expected_input {+ let expected_input = size_for_shape(&[batch_size, layer.cin, seq_len], data_type);+ if input.size() != expected_input {return Err(AudioError::InvalidTokenShape {expected_tokens: expected_input,- actual_tokens: input.num_elements(),+ actual_tokens: input.size(),});}- let data_type = input.data_type();- if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {- return Err(AudioError::Runtime("causal conv1d dtype mismatch".to_string()));- }Ok(())}@@ -114,9 +95,10 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(lengths_array: &Array<B>,batch_size: usize,seq_len: usize,+ data_type: DataType,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {- validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;+ validate_grouped_conv_params(input, data_type, layer, lengths, batch_size, seq_len)?;let cin_i32 = usize_to_i32(layer.cin, "cin")?;let cout_i32 = usize_to_i32(layer.cout, "cout")?;@@ -125,23 +107,16 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;let input_layout_i32 = input_layout.as_i32();let batch_i32 = usize_to_i32(batch_size, "batch_size")?;+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();if layer.groups == 1 {- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let weight_buffer = layer.weight.buffer();- let weight_buffer = weight_buffer.borrow();- let bias_buffer = layer.bias.buffer();- let bias_buffer = bias_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();kernels.causal_conv1d.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input.allocation(),+ layer.weight.allocation(),+ layer.bias.allocation(),+ &mut output,+ lengths_array.allocation(),cin_i32,cout_i32,seq_len_i32,@@ -153,22 +128,12 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>();} else {let groups_i32 = usize_to_i32(layer.groups, "groups")?;- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let weight_buffer = layer.weight.buffer();- let weight_buffer = weight_buffer.borrow();- let bias_buffer = layer.bias.buffer();- let bias_buffer = bias_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();kernels.causal_conv1d_grouped.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input.allocation(),+ layer.weight.allocation(),+ layer.bias.allocation(),+ &mut output,+ lengths_array.allocation(),cin_i32,cout_i32,seq_len_i32,@@ -181,7 +146,7 @@ pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>();}- Ok(output)+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(@@ -194,14 +159,15 @@ pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(lengths_array: &Array<B>,batch_size: usize,seq_len: usize,+ data_type: DataType,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {- validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;- let expected_residual = checked_product(&[batch_size, layer.cout, seq_len])?;- if residual.num_elements() != expected_residual {+ validate_grouped_conv_params(input, data_type, layer, lengths, batch_size, seq_len)?;+ let expected_residual = size_for_shape(&[batch_size, layer.cout, seq_len], data_type);+ if residual.size() != expected_residual {return Err(AudioError::InvalidTokenShape {expected_tokens: expected_residual,- actual_tokens: residual.num_elements(),+ actual_tokens: residual.size(),});}let cin_i32 = usize_to_i32(layer.cin, "cin")?;@@ -211,30 +177,16 @@ pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;let groups_i32 = usize_to_i32(layer.groups, "groups")?;let batch_i32 = usize_to_i32(batch_size, "batch_size")?;- let data_type = input.data_type();- if residual.data_type() != data_type {- return Err(AudioError::Runtime("causal conv1d residual dtype mismatch".to_string()));- }-- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let residual_buffer = residual.buffer();- let residual_buffer = residual_buffer.borrow();- let weight_buffer = layer.weight.buffer();- let weight_buffer = weight_buffer.borrow();- let bias_buffer = layer.bias.buffer();- let bias_buffer = bias_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();kernels.causal_conv1d_grouped_residual.encode(- &*input_buffer,- &*residual_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input.allocation(),+ residual.allocation(),+ layer.weight.allocation(),+ layer.bias.allocation(),+ &mut output,+ lengths_array.allocation(),cin_i32,cout_i32,seq_len_i32,@@ -245,7 +197,7 @@ pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(encoder,);- Ok(output)+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(@@ -259,6 +211,7 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(seq_len_out: usize,input_layout: SequenceLayout,lengths_array: &Array<B>,+ data_type: DataType,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {if lengths.len() != batch_size {@@ -267,35 +220,25 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(actual_lengths: lengths.len(),});}- let expected_input = checked_product(&[batch_size, layer.cin, seq_len_in])?;- if input.num_elements() != expected_input {+ let expected_input = size_for_shape(&[batch_size, layer.cin, seq_len_in], data_type);+ if input.size() != expected_input {return Err(AudioError::InvalidTokenShape {expected_tokens: expected_input,- actual_tokens: input.num_elements(),+ actual_tokens: input.size(),});}if layer.groups == 0 || layer.cin % layer.groups != 0 || layer.cout % layer.groups != 0 {return Err(AudioError::InvalidTokenCardinality);}let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];- if layer.weight.shape() != expected_weight_shape {+ if layer.weight.size() != size_for_shape(&expected_weight_shape, data_type) {return Err(AudioError::Runtime(format!(- "causal transpose weight shape mismatch: expected {:?}, got {:?}",- expected_weight_shape,- layer.weight.shape()+ "causal transpose weight shape mismatch: expected {:?}",+ expected_weight_shape)));}- if layer.bias.shape() != [layer.cout] {- return Err(AudioError::Runtime(format!(- "causal transpose bias shape mismatch: expected [{}], got {:?}",- layer.cout,- layer.bias.shape()- )));- }-- let data_type = input.data_type();- if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {- return Err(AudioError::Runtime("causal transpose dtype mismatch".to_string()));+ if layer.bias.size() != size_for_shape(&[layer.cout], data_type) {+ return Err(AudioError::Runtime(format!("causal transpose bias shape mismatch: expected [{}]", layer.cout)));}let cin_i32 = usize_to_i32(layer.cin, "cin")?;@@ -307,23 +250,15 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(let groups_i32 = usize_to_i32(layer.groups, "groups")?;let input_layout_i32 = input_layout.as_i32();let batch_i32 = usize_to_i32(batch_size, "batch_size")?;-- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let weight_buffer = layer.weight.buffer();- let weight_buffer = weight_buffer.borrow();- let bias_buffer = layer.bias.buffer();- let bias_buffer = bias_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();kernels.causal_conv_transpose1d_causal_pad.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input.allocation(),+ layer.weight.allocation(),+ layer.bias.allocation(),+ &mut output,+ lengths_array.allocation(),cin_i32,cout_i32,seq_in_i32,@@ -336,7 +271,7 @@ pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(encoder,);- Ok(output)+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(@@ -348,6 +283,7 @@ pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(lengths_array: &Array<B>,batch_size: usize,seq_len: usize,+ data_type: DataType,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {if lengths.len() != batch_size {@@ -356,55 +292,37 @@ pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(actual_lengths: lengths.len(),});}- let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;- if input.num_elements() != expected_input {- return Err(AudioError::Runtime(format!(- "pointwise conv input shape mismatch: expected {expected_input} elements ([batch={batch_size}, cin={}, seq_len={seq_len}]), got {}",- layer.cin,- input.num_elements()- )));- }- if layer.weight.shape() != [layer.cout, layer.cin, 1] {+ let expected_input = size_for_shape(&[batch_size, layer.cin, seq_len], data_type);+ if input.size() != expected_input {return Err(AudioError::Runtime(format!(- "pointwise conv weight shape mismatch: expected [{}, {}, 1], got {:?}",- layer.cout,+ "pointwise conv input shape mismatch: expected {expected_input} bytes ([batch={batch_size}, cin={}, seq_len={seq_len}]), got {}",layer.cin,- layer.weight.shape()+ input.size())));}- if layer.bias.shape() != [layer.cout] {+ if layer.weight.size() != size_for_shape(&[layer.cout, layer.cin], data_type) {return Err(AudioError::Runtime(format!(- "pointwise conv bias shape mismatch: expected [{}], got {:?}",- layer.cout,- layer.bias.shape()+ "pointwise conv weight shape mismatch: expected [{}, {}]",+ layer.cout, layer.cin)));}-- let data_type = input.data_type();- if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {- return Err(AudioError::Runtime("pointwise conv dtype mismatch".to_string()));+ if layer.bias.size() != size_for_shape(&[layer.cout], data_type) {+ return Err(AudioError::Runtime(format!("pointwise conv bias shape mismatch: expected [{}]", layer.cout)));}let cin_i32 = usize_to_i32(layer.cin, "cin")?;let cout_i32 = usize_to_i32(layer.cout, "cout")?;let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;let batch_i32 = usize_to_i32(batch_size, "batch_size")?;- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let weight_buffer = layer.weight.buffer();- let weight_buffer = weight_buffer.borrow();- let bias_buffer = layer.bias.buffer();- let bias_buffer = bias_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();kernels.conv1d.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input.allocation(),+ layer.weight.allocation(),+ layer.bias.allocation(),+ &mut output,+ lengths_array.allocation(),cin_i32,cout_i32,seq_len_i32,@@ -418,7 +336,7 @@ pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(encoder,);- Ok(output)+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}pub(super) fn norm_ncs_enqueue<B: Backend>(@@ -431,6 +349,7 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(batch_size: usize,channels: usize,seq_len: usize,+ data_type: DataType,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {if lengths.len() != batch_size {@@ -439,32 +358,21 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(actual_lengths: lengths.len(),});}- if norm.scales.shape() != [channels] {- return Err(AudioError::Runtime(format!(- "norm scale shape mismatch: expected [{channels}], got {:?}",- norm.scales.shape()- )));+ if norm.scales.size() != size_for_shape(&[channels], data_type) {+ return Err(AudioError::Runtime(format!("norm scale shape mismatch: expected [{channels}]")));}- if norm.bias.shape() != [channels] {- return Err(AudioError::Runtime(format!(- "norm bias shape mismatch: expected [{channels}], got {:?}",- norm.bias.shape()- )));+ if norm.bias.size() != size_for_shape(&[channels], data_type) {+ return Err(AudioError::Runtime(format!("norm bias shape mismatch: expected [{channels}]")));}- let expected_input = checked_product(&[batch_size, channels, seq_len])?;- if input.num_elements() != expected_input {+ let expected_input = size_for_shape(&[batch_size, channels, seq_len], data_type);+ if input.size() != expected_input {return Err(AudioError::InvalidTokenShape {expected_tokens: expected_input,- actual_tokens: input.num_elements(),+ actual_tokens: input.size(),});}- let data_type = input.data_type();- if norm.scales.data_type() != data_type || norm.bias.data_type() != data_type {- return Err(AudioError::Runtime("norm dtype mismatch".to_string()));- }-let channels_i32 = usize_to_i32(channels, "channels")?;let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;let batch_i32 = usize_to_i32(batch_size, "batch_size")?;@@ -473,22 +381,15 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(} else {0};- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let scales_buffer = norm.scales.buffer();- let scales_buffer = scales_buffer.borrow();- let bias_buffer = norm.bias.buffer();- let bias_buffer = bias_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- let lengths_buffer = lengths_array.buffer();- let lengths_buffer = lengths_buffer.borrow();+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();kernels.norm_ncs.encode(- &*input_buffer,- &*scales_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input.allocation(),+ norm.scales.allocation(),+ norm.bias.allocation(),+ &mut output,+ lengths_array.allocation(),channels_i32,seq_len_i32,norm.epsilon,@@ -497,7 +398,7 @@ pub(super) fn norm_ncs_enqueue<B: Backend>(encoder,);- Ok(output)+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}pub(super) fn gelu_enqueue<B: Backend>(@@ -508,12 +409,11 @@ pub(super) fn gelu_enqueue<B: Backend>() -> AudioResult<Array<B>> {let n_u32 = u32::try_from(input.num_elements()).map_err(|_| AudioError::Runtime("gelu element count exceeds u32 range".to_string()))?;- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::GELU, encoder);- Ok(output)+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();+ kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, ActivationType::GELU, encoder);+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}pub(super) fn add_enqueue<B: Backend>(@@ -523,29 +423,15 @@ pub(super) fn add_enqueue<B: Backend>(output: Array<B>,kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {- if a.num_elements() != b.num_elements() {- return Err(AudioError::Runtime(format!(- "elementwise add shape mismatch: {} vs {}",- a.num_elements(),- b.num_elements()- )));- }- if a.data_type() != b.data_type() {- return Err(AudioError::Runtime(format!(- "elementwise add dtype mismatch: {:?} vs {:?}",- a.data_type(),- b.data_type()- )));+ if a.size() != b.size() {+ return Err(AudioError::Runtime(format!("elementwise add shape mismatch: {} vs {}", a.size(), b.size())));}let n_i32 = usize_to_i32(a.num_elements(), "n")?;- let a_buffer = a.buffer();- let a_buffer = a_buffer.borrow();- let b_buffer = b.buffer();- let b_buffer = b_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- kernels.add.encode(&*a_buffer, &*b_buffer, &mut *output_buffer, n_i32, encoder);- Ok(output)+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();+ kernels.add.encode(a.allocation(), b.allocation(), &mut output, n_i32, encoder);+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}pub(super) fn tanh_enqueue<B: Backend>(@@ -556,10 +442,9 @@ pub(super) fn tanh_enqueue<B: Backend>() -> AudioResult<Array<B>> {let n_u32 = u32::try_from(input.num_elements()).map_err(|_| AudioError::Runtime("tanh element count exceeds u32 range".to_string()))?;- let input_buffer = input.buffer();- let input_buffer = input_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::TANH, encoder);- Ok(output)+ let output_shape = output.shape().to_vec();+ let output_data_type = output.data_type();+ let mut output = output.into_allocation();+ kernels.activation.encode(Some(input.allocation()), &mut output, n_u32, ActivationType::TANH, encoder);+ Ok(unsafe { Array::from_allocation(output, 0, &output_shape, output_data_type) })}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rsindex 1ef48c5d3..df09c9349 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs@@ -1,5 +1,5 @@use super::*;-use crate::backends::common::Buffer;+use crate::array::{Array, ArrayContextExt};pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {match data_type {@@ -12,8 +12,7 @@ pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {pub(super) struct StructuredAudioPostModuleRuntime<B: Backend> {pub(super) context: Rc<B::Context>,pub(super) model_shape: ModelShape,- pub(super) scratch_buffers: ScratchBuffers<B>,- pub(super) shared_buffers: Rc<RefCell<SharedBuffers<B>>>,+ pub(super) shared_buffers: Rc<SharedBuffers<B>>,pub(super) layers: Box<[LayerExecutables<B>]>,pub(super) output_norm: RMSNorm<B>,pub(super) max_sequence_length: usize,@@ -46,134 +45,46 @@ pub(super) struct FishAudioQuantizerResources<B: Backend> {pub(super) residual_out_bias: Array<B>,}-// TODO: once `Array<B>` is bridged to `backends::common::allocator::Allocation`-// project-wide, replace this whole workspace with `Encoder::allocate_scratch`.-// That pool already handles auto-expand on demand, range aliasing for-// non-overlapping lifetimes, and free-on-drop tied to command-buffer-// completion — i.e. everything this struct reimplements by hand.pub(super) struct DecodeWorkspace<B: Backend> {- slots: RefCell<Vec<Rc<RefCell<B::DenseBuffer>>>>,- next_search: Cell<usize>,- min_buffer_size: Cell<usize>,- lengths_pair: [RefCell<Option<Array<B>>>; 2],+ _marker: std::marker::PhantomData<B>,}impl<B: Backend> DecodeWorkspace<B> {fn new() -> Self {Self {- slots: RefCell::new(Vec::new()),- next_search: Cell::new(0),- min_buffer_size: Cell::new(0),- lengths_pair: [RefCell::new(None), RefCell::new(None)],+ _marker: std::marker::PhantomData,}}- pub(super) fn prime_for(- &self,- context: &B::Context,- bytes: usize,- ) -> AudioResult<()> {- if bytes <= self.min_buffer_size.get() {- return Ok(());- }- self.min_buffer_size.set(bytes);- let mut slots = self.slots.borrow_mut();- for slot in slots.iter_mut() {- if Rc::strong_count(slot) == 1 && slot.borrow().size() < bytes {- *slot = create_labelled_buffer::<B>(context, bytes, "structured_audio_scratch")?;- }- }- Ok(())- }-pub(super) fn lengths_array(&self,- context: &B::Context,- index: usize,+ encoder: &mut Encoder<B>,min_elements: usize,) -> Array<B> {- debug_assert!(index < 2);- let mut slot = self.lengths_pair[index].borrow_mut();- if let Some(existing) = slot.as_ref() {- if existing.num_elements() >= min_elements {- return existing.clone();- }- }- let label = if index == 0 {- "structured_audio_lengths_a"- } else {- "structured_audio_lengths_b"- };- let array = context.create_array_zeros(&[min_elements], DataType::I32, label);- *slot = Some(array.clone());- array+ let allocation = encoder+ .allocate_constant(min_elements * std::mem::size_of::<i32>())+ .expect("Failed to allocate structured audio lengths");+ unsafe { Array::from_allocation(allocation, 0, &[min_elements], DataType::I32) }}pub(super) fn next_scratch(&self,- context: &B::Context,+ encoder: &mut Encoder<B>,shape: &[usize],data_type: DataType,- label: &str,- ) -> AudioResult<Array<B>> {- let needed = size_for_shape(shape, data_type).max(self.min_buffer_size.get());- let mut slots = self.slots.borrow_mut();- let slot_count = slots.len();-- if slot_count > 0 {- let start = self.next_search.get() % slot_count;- for offset in 0..slot_count {- let slot_index = (start + offset) % slot_count;- if Rc::strong_count(&slots[slot_index]) == 1 && slots[slot_index].borrow().size() >= needed {- self.next_search.set((slot_index + 1) % slot_count);- return Ok(unsafe { Array::from_parts(slots[slot_index].clone(), 0, shape, data_type) });- }- }- for offset in 0..slot_count {- let slot_index = (start + offset) % slot_count;- if Rc::strong_count(&slots[slot_index]) == 1 {- slots[slot_index] = create_labelled_buffer::<B>(context, needed, label)?;- self.next_search.set((slot_index + 1) % slot_count);- return Ok(unsafe { Array::from_parts(slots[slot_index].clone(), 0, shape, data_type) });- }- }- }-- let buffer_rc = create_labelled_buffer::<B>(context, needed, label)?;- slots.push(buffer_rc.clone());- self.next_search.set(0);- Ok(unsafe { Array::from_parts(buffer_rc, 0, shape, data_type) })- }-- pub(super) fn reset(&self) {- self.slots.borrow_mut().clear();- self.next_search.set(0);- self.min_buffer_size.set(0);- *self.lengths_pair[0].borrow_mut() = None;- *self.lengths_pair[1].borrow_mut() = None;+ ) -> Array<B> {+ let allocation =+ encoder.allocate_scratch(size_for_shape(shape, data_type)).expect("Failed to allocate scratch");+ unsafe { Array::from_allocation(allocation, 0, shape, data_type) }}}-fn create_labelled_buffer<B: Backend>(- context: &B::Context,- bytes: usize,- label: &str,-) -> AudioResult<Rc<RefCell<B::DenseBuffer>>> {- let mut buffer = context- .create_buffer(bytes)- .map_err(|err| AudioError::Runtime(format!("failed to create scratch buffer '{label}': {err}")))?;- buffer.set_label(Some(label));- Ok(Rc::new(RefCell::new(buffer)))-}-pub(in crate::audio::nanocodec::runtime) struct StructuredAudioRuntimeResources<B: Backend> {context: Rc<B::Context>,pub(super) kernels_by_dtype: RefCell<HashMap<u8, Rc<StructuredAudioKernelCache<B>>>>,pub(super) post_module_runtime: RefCell<Option<Rc<StructuredAudioPostModuleRuntime<B>>>>,pub(super) vocoder_graph: RefCell<Option<Rc<StructuredAudioDecoderGraph<B>>>>,pub(super) quantizer_resources: RefCell<Option<Rc<FishAudioQuantizerResources<B>>>>,- token_staging: RefCell<Option<Array<B>>>,- length_staging: RefCell<Option<Array<B>>>,pub(super) decode_workspace: DecodeWorkspace<B>,}@@ -185,8 +96,6 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {post_module_runtime: RefCell::new(None),vocoder_graph: RefCell::new(None),quantizer_resources: RefCell::new(None),- token_staging: RefCell::new(None),- length_staging: RefCell::new(None),decode_workspace: DecodeWorkspace::new(),}}@@ -195,15 +104,6 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {&self.context}- /// Reset the decode workspace and staging arrays so that subsequent- /// decode passes allocate fresh buffers. Call this after submitting a- /// command buffer whose encoded work references the current buffers.- pub(in crate::audio::nanocodec::runtime) fn reset_for_pending(&self) {- self.decode_workspace.reset();- *self.token_staging.borrow_mut() = None;- *self.length_staging.borrow_mut() = None;- }-pub(super) fn kernels(&self,data_type: DataType,@@ -221,30 +121,14 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {&self,min_elements: usize,) -> Array<B> {- let mut slot = self.token_staging.borrow_mut();- if let Some(existing) = slot.as_ref() {- if existing.num_elements() >= min_elements {- return existing.clone();- }- }- let array = self.context.create_array_zeros(&[min_elements], DataType::U32, "quantizer_token_staging");- *slot = Some(array.clone());- array+ self.context.create_array_uninitialized(&[min_elements], DataType::U32)}pub(super) fn length_staging(&self,min_elements: usize,) -> Array<B> {- let mut slot = self.length_staging.borrow_mut();- if let Some(existing) = slot.as_ref() {- if existing.num_elements() >= min_elements {- return existing.clone();- }- }- let array = self.context.create_array_zeros(&[min_elements], DataType::I32, "quantizer_length_staging");- *slot = Some(array.clone());- array+ self.context.create_array_uninitialized(&[min_elements], DataType::I32)}}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex 53f5fdf31..4b871eabd 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -1,4 +1,10 @@use super::*;+use crate::{+ array::{Array, ArrayContextExt},+ backends::common::{Allocation, AsBufferRangeRef},+ encodable_block::LayerArguments,+ forward_pass::token_inputs::TokenInputs,+};impl StructuredAudioCodecGraph {pub(super) fn apply_convnext_ncs_enqueued<B: Backend>(@@ -15,58 +21,65 @@ impl StructuredAudioCodecGraph {kernels: &StructuredAudioKernelCache<B>,) -> AudioResult<Array<B>> {let ws = &resources.decode_workspace;- let ctx = resources.context();- let data_type = input.data_type();- let residual = input.clone();- let mut x = causal_conv1d_grouped_enqueue(+ let cnxt_dw = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);+ let x = causal_conv1d_grouped_enqueue(encoder,input,- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_dw")?,+ cnxt_dw,&layer.depthwise_conv,SequenceLayout::Ncs,lengths,lengths_array,batch_size,seq_len,+ self.vocoder_data_type,kernels,)?;- x = norm_ncs_enqueue(+ let cnxt_norm = ws.next_scratch(encoder, &[batch_size, channels, seq_len], self.vocoder_data_type);+ let x = norm_ncs_enqueue(encoder,&x,- ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_norm")?,+ cnxt_norm,&layer.norm,lengths,lengths_array,batch_size,channels,seq_len,+ self.vocoder_data_type,kernels,)?;- x = conv1d_pointwise_ncs_enqueue(+ let cnxt_pw1 = ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);+ let x = conv1d_pointwise_ncs_enqueue(encoder,&x,- ws.next_scratch(ctx, &[batch_size, layer.pwconv1.cout, seq_len], data_type, "cnxt_pw1")?,+ cnxt_pw1,&layer.pwconv1,lengths,lengths_array,batch_size,seq_len,+ self.vocoder_data_type,kernels,)?;- x = gelu_enqueue(encoder, &x, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_gelu")?, kernels)?;- x = conv1d_pointwise_ncs_enqueue(+ let cnxt_gelu = ws.next_scratch(encoder, &[batch_size, layer.pwconv1.cout, seq_len], self.vocoder_data_type);+ let x = gelu_enqueue(encoder, &x, cnxt_gelu, kernels)?;+ let cnxt_pw2 = ws.next_scratch(encoder, &[batch_size, layer.pwconv2.cout, seq_len], self.vocoder_data_type);+ let x = conv1d_pointwise_ncs_enqueue(encoder,&x,- ws.next_scratch(ctx, &[batch_size, layer.pwconv2.cout, seq_len], data_type, "cnxt_pw2")?,+ cnxt_pw2,&layer.pwconv2,lengths,lengths_array,batch_size,seq_len,+ self.vocoder_data_type,kernels,)?;- add_enqueue(encoder, &x, &residual, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_add")?, kernels)+ let cnxt_add = ws.next_scratch(encoder, &[batch_size, layer.pwconv2.cout, seq_len], self.vocoder_data_type);+ add_enqueue(encoder, &x, input, cnxt_add, kernels)}pub(super) fn build_post_module_runtime<B: Backend>(@@ -103,12 +116,11 @@ impl StructuredAudioCodecGraph {.map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;let max_sequence_length = decoder_config.context_length.max(required_sequence_length.max(1));- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);{let transformer_tree = root_loader_view.subtree(transformer_subtree_name).map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;- let mut shared_buffers = shared_buffers.borrow_mut();if let Some(global_rope) = &mut shared_buffers.global_rope {global_rope.update_data(&transformer_tree, "global_rope");}@@ -116,7 +128,7 @@ impl StructuredAudioCodecGraph {local_rope.update_data(&transformer_tree, "local_rope");}}- let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_sequence_length);+ let shared_buffers = Rc::new(shared_buffers);let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(context.as_ref(),&decoder_config,@@ -127,7 +139,6 @@ impl StructuredAudioCodecGraph {Ok(StructuredAudioPostModuleRuntime {context,model_shape,- scratch_buffers,shared_buffers,layers,output_norm,@@ -180,7 +191,7 @@ impl StructuredAudioCodecGraph {self.config.codebook_dim,data_type,)?;- let codebook_dim = semantic_codebook.shape()[1];+ let codebook_dim = self.config.codebook_dim;let semantic_out_proj = read_float_matrix_exact::<B>(&semantic_tree.subtree("out_proj")?,"weights",@@ -194,21 +205,11 @@ impl StructuredAudioCodecGraph {let residual_quantizers = self.config.n_codebooks;let residual_count_for_shape = residual_quantizers.max(1);let residual_codebook_rows_for_shape = self.codebook_size.max(1);- let residual_codebooks = context.create_array_zeros(- &[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim],- data_type,- "structured_audio_quantizer_residual_codebooks",- );- let residual_out_proj = context.create_array_zeros(- &[residual_count_for_shape, self.input_dim, codebook_dim],- data_type,- "structured_audio_quantizer_residual_out_proj",- );- let residual_out_bias = context.create_array_zeros(- &[residual_count_for_shape, self.input_dim],- data_type,- "structured_audio_quantizer_residual_out_bias",- );+ let mut residual_codebooks = context+ .create_array_zeros(&[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim], data_type);+ let mut residual_out_proj =+ context.create_array_zeros(&[residual_count_for_shape, self.input_dim, codebook_dim], data_type);+ let mut residual_out_bias = context.create_array_zeros(&[residual_count_for_shape, self.input_dim], data_type);let residual_root = quantizer_tree.subtree("quantizer")?.subtree("quantizers")?;for index in 0..residual_quantizers {@@ -234,12 +235,9 @@ impl StructuredAudioCodecGraph {data_type,)?;- let mut dst_codebook = outer_axis_view(&residual_codebooks, index, &[self.codebook_size, codebook_dim])?;- dst_codebook.copy_from_array(&codebook);- let mut dst_out_proj = outer_axis_view(&residual_out_proj, index, &[self.input_dim, codebook_dim])?;- dst_out_proj.copy_from_array(&out_proj);- let mut dst_out_bias = outer_axis_view(&residual_out_bias, index, &[self.input_dim])?;- dst_out_bias.copy_from_array(&out_bias);+ copy_to_outer_axis_slice(&mut residual_codebooks, index, &codebook, &[self.codebook_size, codebook_dim])?;+ copy_to_outer_axis_slice(&mut residual_out_proj, index, &out_proj, &[self.input_dim, codebook_dim])?;+ copy_to_outer_axis_slice(&mut residual_out_bias, index, &out_bias, &[self.input_dim])?;}Ok(FishAudioQuantizerResources {@@ -298,100 +296,82 @@ impl StructuredAudioCodecGraph {pub(super) fn encode_post_module_layers<B: Backend>(runtime: &StructuredAudioPostModuleRuntime<B>,- state: &mut ForwardPassState<B>,+ token_inputs: &TokenInputs<B>,+ batch_dim: usize,+ main: Allocation<B>,encoder: &mut Encoder<B>,- ) -> AudioResult<()> {+ ) -> AudioResult<Allocation<B>> {let encoding_parameters = EncodingParameters::new();+ let mut main = main;+ let mut shortcut = encoder+ .allocate_scratch(main.as_buffer_range_ref().range().len())+ .map_err(|err| AudioError::Runtime(format!("post_module shortcut allocation failed: {err}")))?;for layer in runtime.layers.iter() {- layer- .encode(state, &encoding_parameters, encoder)+ let rope_type = layer.rope_type();+ main = layer+ .encode(+ LayerArguments {+ batch_dim,+ token_positions: token_inputs.token_positions(),+ token_parents: token_inputs.token_parents(),+ token_subtrie_ranges: None,+ attention_sinks: runtime.shared_buffers.attention_sinks(layer.layer_index),+ rope_cosines: rope_type.and_then(|rope_type| runtime.shared_buffers.rope_cosines(rope_type)),+ rope_sines: rope_type.and_then(|rope_type| runtime.shared_buffers.rope_sines(rope_type)),+ rope_max_sequence_length: runtime.model_shape.context_length(),+ rope_dim: runtime.model_shape.rope_dim(),+ sampling_start: 0,+ sampling_length: batch_dim,+ cache_layer: None,+ #[cfg(feature = "tracing")]+ trace: None,+ },+ &encoding_parameters,+ main,+ &mut shortcut,+ encoder,+ ).map_err(|err| AudioError::Runtime(format!("post_module layer encode failed: {err}")))?;}runtime.output_norm- .encode(state, encoder)- .map_err(|err| AudioError::Runtime(format!("post_module output norm encode failed: {err}")))?;- Ok(())+ .encode(&main, 0, batch_dim, Some(&mut shortcut), encoder)+ .map_err(|err| AudioError::Runtime(format!("post_module output norm encode failed: {err}")))}pub(super) fn apply_post_module_single_batch_enqueued<B: Backend>(&self,resources: &StructuredAudioRuntimeResources<B>,encoder: &mut Encoder<B>,- latent_nsc: &Array<B>,+ latent_nsc: Array<B>,frames: usize,) -> AudioResult<Array<B>> {if frames == 0 {- return Ok(latent_nsc.clone());+ return Ok(latent_nsc);}- let expected_elements = checked_product(&[frames, self.input_dim])?;- if latent_nsc.num_elements() != expected_elements {+ let expected_size = size_for_shape(&[frames, self.input_dim], self.vocoder_data_type);+ if latent_nsc.size() != expected_size {return Err(AudioError::InvalidTokenShape {- expected_tokens: expected_elements,- actual_tokens: latent_nsc.num_elements(),+ expected_tokens: expected_size,+ actual_tokens: latent_nsc.size(),});}let runtime = self.post_module_runtime(resources, frames.max(1))?;let token_ids = vec![0_u64; frames];let token_positions = (0..frames).collect::<Vec<_>>();- let mut state = ForwardPassState::new_classifier(- runtime.context.clone(),- &runtime.model_shape,- &runtime.scratch_buffers,- runtime.shared_buffers.clone(),- &token_ids,- &token_positions,- 1,- );-- let main = state.array(ArrayId::Main);- let main_output = {- if main.shape() != [frames, self.input_dim] {- return Err(AudioError::Runtime(format!(- "post_module main shape mismatch: expected [{frames}, {}], got {:?}",- self.input_dim,- main.shape()- )));- }- if main.data_type() != latent_nsc.data_type() {- return Err(AudioError::Runtime(format!(- "post_module dtype mismatch: main={:?}, latent={:?}",- main.data_type(),- latent_nsc.data_type()- )));- }- main.clone()- };-- let copy_bytes = latent_nsc- .num_elements()- .checked_mul(latent_nsc.data_type().size_in_bytes())- .ok_or(AudioError::Runtime("post_module copy size overflow".to_string()))?;- {- let latent_buffer = latent_nsc.buffer();- let main_output_buffer = main_output.buffer();- let latent_buffer = latent_buffer.borrow();- let mut main_output_buffer = main_output_buffer.borrow_mut();- encoder.encode_copy(- &latent_buffer,- latent_nsc.offset()..latent_nsc.offset() + copy_bytes,- &mut main_output_buffer,- main_output.offset()..main_output.offset() + copy_bytes,- );- }- Self::encode_post_module_layers(&runtime, &mut state, encoder)?;-- Ok(main_output)+ let token_inputs = TokenInputs::new_classifier(runtime.context.as_ref(), &token_ids, &token_positions);+ let main =+ Self::encode_post_module_layers(&runtime, &token_inputs, frames, latent_nsc.into_allocation(), encoder)?;+ Ok(unsafe { Array::from_allocation(main, 0, &[frames, self.input_dim], self.vocoder_data_type) })}pub(super) fn apply_post_module_enqueued<B: Backend>(&self,resources: &StructuredAudioRuntimeResources<B>,- context: &Rc<B::Context>,encoder: &mut Encoder<B>,- latent_nsc: &Array<B>,+ latent_nsc: Array<B>,lengths: &[usize],batch_size: usize,frames: usize,@@ -402,22 +382,13 @@ impl StructuredAudioCodecGraph {actual_lengths: lengths.len(),});}- let expected_elements = checked_product(&[batch_size, frames, self.input_dim])?;- if latent_nsc.num_elements() != expected_elements {+ let expected_size = size_for_shape(&[batch_size, frames, self.input_dim], self.vocoder_data_type);+ if latent_nsc.size() != expected_size {return Err(AudioError::InvalidTokenShape {- expected_tokens: expected_elements,- actual_tokens: latent_nsc.num_elements(),+ expected_tokens: expected_size,+ actual_tokens: latent_nsc.size(),});}- if batch_size == 1 && lengths.first().copied() == Some(frames) {- return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);- }-- let output = context.create_array_zeros(- &[batch_size, frames, self.input_dim],- latent_nsc.data_type(),- "structured_audio_post_module_output_nsc",- );let mut batch_indices_by_length = BTreeMap::<usize, Vec<usize>>::new();for (batch_index, &active_len) in lengths.iter().enumerate() {if active_len == 0 {@@ -431,94 +402,84 @@ impl StructuredAudioCodecGraph {}batch_indices_by_length.entry(active_len).or_default().push(batch_index);}+ if frames == 0 {+ return Ok(latent_nsc);+ }+ if batch_size == 1 && lengths.first().copied() == Some(frames) {+ return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);+ }+ let mut output = encoder.allocate_scratch(expected_size).map_err(|err| {+ AudioError::Runtime(format!("failed to allocate structured audio post_module output: {err}"))+ })?;if batch_indices_by_length.is_empty() {- let mut output = output;- output.copy_from_array(latent_nsc);- return Ok(output);+ encoder.encode_copy(+ latent_nsc.allocation(),+ latent_nsc.offset()..latent_nsc.offset() + latent_nsc.size(),+ &mut output,+ 0..latent_nsc.size(),+ );+ return Ok(unsafe {+ Array::from_allocation(output, 0, &[batch_size, frames, self.input_dim], self.vocoder_data_type)+ });}let full_copy_bytes = latent_nsc.size();+ let batch_stride_bytes = size_for_shape(&[frames, self.input_dim], self.vocoder_data_type);let mut copied_output_prefix = false;for (active_len, batch_indices) in batch_indices_by_length {let runtime = self.post_module_runtime(resources, active_len.max(1))?;let token_ids = vec![0_u64; active_len];let token_positions = (0..active_len).collect::<Vec<_>>();- let mut state = ForwardPassState::new_classifier(- runtime.context.clone(),- &runtime.model_shape,- &runtime.scratch_buffers,- runtime.shared_buffers.clone(),- &token_ids,- &token_positions,- 1,- );+ let token_inputs = TokenInputs::new_classifier(runtime.context.as_ref(), &token_ids, &token_positions);- let main = state.array(ArrayId::Main);- let main_output = {- if main.shape() != [active_len, self.input_dim] {- return Err(AudioError::Runtime(format!(- "post_module main shape mismatch: expected [{active_len}, {}], got {:?}",- self.input_dim,- main.shape()- )));- }- if main.data_type() != latent_nsc.data_type() {- return Err(AudioError::Runtime(format!(- "post_module dtype mismatch: main={:?}, latent={:?}",- main.data_type(),- latent_nsc.data_type()- )));- }- main.clone()- };+ let main_shape = runtime.model_shape.main_shape(active_len);+ if main_shape != [active_len, self.input_dim] {+ return Err(AudioError::Runtime(format!(+ "post_module main shape mismatch: expected [{active_len}, {}], got [{}, {}]",+ self.input_dim, main_shape[0], main_shape[1]+ )));+ }+ if runtime.model_shape.activation_data_type() != self.vocoder_data_type {+ return Err(AudioError::Runtime(format!(+ "post_module dtype mismatch: main={:?}, latent={:?}",+ runtime.model_shape.activation_data_type(),+ self.vocoder_data_type+ )));+ }for &batch_index in &batch_indices {if !copied_output_prefix {- {- let latent_buffer = latent_nsc.buffer();- let output_buffer = output.buffer();- let latent_buffer = latent_buffer.borrow();- let mut output_buffer = output_buffer.borrow_mut();- encoder.encode_copy(- &latent_buffer,- latent_nsc.offset()..latent_nsc.offset() + full_copy_bytes,- &mut output_buffer,- output.offset()..output.offset() + full_copy_bytes,- );- }- copied_output_prefix = true;- }- let source = array_batch_view(latent_nsc, batch_index, frames, self.input_dim, active_len)?;- {- let source_buffer = source.buffer();- let main_output_buffer = main_output.buffer();- let source_buffer = source_buffer.borrow();- let mut main_output_buffer = main_output_buffer.borrow_mut();- encoder.encode_copy(- &source_buffer,- source.offset()..source.offset() + source.size(),- &mut main_output_buffer,- main_output.offset()..main_output.offset() + source.size(),- );- }- Self::encode_post_module_layers(&runtime, &mut state, encoder)?;- let destination = array_batch_view(&output, batch_index, frames, self.input_dim, active_len)?;- {- let main_output_buffer = main_output.buffer();- let destination_buffer = destination.buffer();- let main_output_buffer = main_output_buffer.borrow();- let mut destination_buffer = destination_buffer.borrow_mut();encoder.encode_copy(- &main_output_buffer,- main_output.offset()..main_output.offset() + destination.size(),- &mut destination_buffer,- destination.offset()..destination.offset() + destination.size(),+ latent_nsc.allocation(),+ latent_nsc.offset()..latent_nsc.offset() + full_copy_bytes,+ &mut output,+ 0..full_copy_bytes,);+ copied_output_prefix = true;}+ let source_range =+ array_batch_byte_range(&latent_nsc, batch_index, frames, self.input_dim, active_len)?;+ let main_array = resources.decode_workspace.next_scratch(+ encoder,+ &[active_len, self.input_dim],+ self.vocoder_data_type,+ );+ let mut main = main_array.into_allocation();+ let source_size = source_range.end - source_range.start;+ encoder.encode_copy(latent_nsc.allocation(), source_range, &mut main, 0..source_size);+ let main = Self::encode_post_module_layers(&runtime, &token_inputs, active_len, main, encoder)?;+ let destination_offset = batch_index * batch_stride_bytes;+ let active_bytes = active_len * self.input_dim * self.vocoder_data_type.size_in_bytes();+ encoder.encode_copy(+ &main,+ 0..active_bytes,+ &mut output,+ destination_offset..destination_offset + active_bytes,+ );}}- Ok(output)+ Ok(unsafe { Array::from_allocation(output, 0, &[batch_size, frames, self.input_dim], self.vocoder_data_type) })}}diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rsindex 249ee8573..5677cdaec 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/shared.rs@@ -1,21 +1,27 @@+use std::ops::Range;+use super::*;+use crate::array::Array;-pub(super) fn array_batch_view<B: Backend>(+pub(super) fn array_batch_byte_range<B: Backend>(array: &Array<B>,batch_index: usize,frames: usize,channels: usize,active_frames: usize,-) -> AudioResult<Array<B>> {+) -> AudioResult<Range<usize>> {let batch_stride_bytes = size_for_shape(&[frames, channels], array.data_type());let batch_offset = batch_index.checked_mul(batch_stride_bytes).and_then(|value| value.checked_add(array.offset()))- .ok_or(AudioError::Runtime("array batch view offset overflow".to_string()))?;+ .ok_or(AudioError::Runtime("array batch range offset overflow".to_string()))?;if active_frames > frames {- return Err(AudioError::Runtime("array batch view active_frames exceeds frames".to_string()));+ return Err(AudioError::Runtime("array batch range active_frames exceeds frames".to_string()));}- Ok(unsafe { Array::from_parts(array.buffer(), batch_offset, &[active_frames, channels], array.data_type()) })+ let byte_len = size_for_shape(&[active_frames, channels], array.data_type());+ let end =+ batch_offset.checked_add(byte_len).ok_or(AudioError::Runtime("array batch range end overflow".to_string()))?;+ Ok(batch_offset..end)}#[derive(Clone, Copy, Debug, PartialEq, Eq)]@@ -45,7 +51,7 @@ pub struct StructuredAudioCodecGraph {pub(in crate::audio::nanocodec::runtime) vocoder_data_type: DataType,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioConv1d<B: Backend> {pub(super) weight: Array<B>,pub(super) bias: Array<B>,@@ -56,7 +62,7 @@ pub(super) struct StructuredAudioConv1d<B: Backend> {pub(super) groups: usize,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioConvTranspose1d<B: Backend> {pub(super) weight: Array<B>,pub(super) bias: Array<B>,@@ -67,7 +73,7 @@ pub(super) struct StructuredAudioConvTranspose1d<B: Backend> {pub(super) groups: usize,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioPointwiseConv<B: Backend> {pub(super) weight: Array<B>,pub(super) bias: Array<B>,@@ -75,7 +81,7 @@ pub(super) struct StructuredAudioPointwiseConv<B: Backend> {pub(super) cout: usize,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioNorm<B: Backend> {pub(super) scales: Array<B>,pub(super) bias: Array<B>,@@ -83,7 +89,7 @@ pub(super) struct StructuredAudioNorm<B: Backend> {pub(super) subtract_mean: bool,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioConvNeXt<B: Backend> {pub(super) depthwise_conv: StructuredAudioConv1d<B>,pub(super) norm: StructuredAudioNorm<B>,@@ -91,7 +97,7 @@ pub(super) struct StructuredAudioConvNeXt<B: Backend> {pub(super) pwconv2: StructuredAudioPointwiseConv<B>,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioResidualUnit<B: Backend> {pub(super) snake1_alpha: Array<B>,pub(super) conv1: StructuredAudioConv1d<B>,@@ -99,7 +105,7 @@ pub(super) struct StructuredAudioResidualUnit<B: Backend> {pub(super) conv2: StructuredAudioConv1d<B>,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioDecoderBlock<B: Backend> {pub(super) snake_alpha: Array<B>,pub(super) trans_conv: StructuredAudioConvTranspose1d<B>,@@ -108,7 +114,7 @@ pub(super) struct StructuredAudioDecoderBlock<B: Backend> {pub(super) res_unit3: StructuredAudioResidualUnit<B>,}-#[derive(Debug, Clone)]+#[derive(Debug)]pub(super) struct StructuredAudioDecoderGraph<B: Backend> {pub(super) first_conv: StructuredAudioConv1d<B>,pub(super) upsample_blocks: Vec<(StructuredAudioConvTranspose1d<B>, StructuredAudioConvNeXt<B>)>,diff --git a/crates/backend-uzu/src/backends/common/allocation_access.rs b/crates/backend-uzu/src/backends/common/allocation_access.rsnew file mode 100644index 000000000..11e98e9cb--- /dev/null+++ b/crates/backend-uzu/src/backends/common/allocation_access.rs@@ -0,0 +1,60 @@+use thiserror::Error;++use crate::{+ ArrayElement,+ backends::common::{Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, DenseBuffer},+};++#[derive(Debug, Clone, PartialEq, Eq, Error)]+pub enum AllocationAccessError {+ #[error("allocation write length {write_len} exceeds allocation length {range_len}")]+ WriteExceedsRange {+ write_len: usize,+ range_len: usize,+ },+}++pub fn allocation_copy_from_slice<B: Backend, T: ArrayElement>(+ allocation: &mut Allocation<B>,+ data: &[T],+) -> Result<(), AllocationAccessError> {+ let bytes = bytemuck::cast_slice(data);+ if bytes.is_empty() {+ return Ok(());+ }+ let destination = allocation_as_bytes_mut(allocation);+ if bytes.len() > destination.len() {+ return Err(AllocationAccessError::WriteExceedsRange {+ write_len: bytes.len(),+ range_len: destination.len(),+ });+ }+ destination[..bytes.len()].copy_from_slice(bytes);+ Ok(())+}++pub fn allocation_as_bytes<B: Backend>(allocation: &Allocation<B>) -> &[u8] {+ let buffer_range = allocation.as_buffer_range_ref();+ let range = buffer_range.range();+ unsafe {+ std::slice::from_raw_parts(+ (buffer_range.buffer().cpu_ptr().as_ptr() as *const u8).add(range.start),+ range.len(),+ )+ }+}++pub fn allocation_as_bytes_mut<B: Backend>(allocation: &mut Allocation<B>) -> &mut [u8] {+ let buffer_range = allocation.as_buffer_range_mut();+ let range = buffer_range.range();+ unsafe {+ std::slice::from_raw_parts_mut(+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),+ range.len(),+ )+ }+}++pub fn allocation_to_vec<B: Backend, T: ArrayElement>(allocation: &Allocation<B>) -> Vec<T> {+ bytemuck::cast_slice(allocation_as_bytes(allocation)).to_vec()+}diff --git a/crates/backend-uzu/src/backends/common/allocator/allocator.rs b/crates/backend-uzu/src/backends/common/allocator/allocator.rsindex 1d50b5374..a24691f26 100644--- a/crates/backend-uzu/src/backends/common/allocator/allocator.rs+++ b/crates/backend-uzu/src/backends/common/allocator/allocator.rs@@ -90,6 +90,7 @@ impl<B: Backend> Allocator<B> {size: usize,allocation_type: AllocationType<B>,) -> Result<Allocation<B>, B::Error> {+ assert!(size > 0, "allocation size must be greater than 0");let alignment = usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, 16_384);let allocation_type = match allocation_type {AllocationType::Global => RangeAllocationType::Global,diff --git a/crates/backend-uzu/src/backends/common/buffer.rs b/crates/backend-uzu/src/backends/common/buffer.rsindex caa0f32d7..bd91dbb0e 100644--- a/crates/backend-uzu/src/backends/common/buffer.rs+++ b/crates/backend-uzu/src/backends/common/buffer.rs@@ -57,6 +57,17 @@ impl<'a, B: Buffer> BufferRangeRef<'a, B> {pub fn range(&self) -> Range<usize> {self.range.clone()}++ pub(crate) fn subrange(+ self,+ range: Range<usize>,+ ) -> Self {+ assert!(range.end <= self.range.len(), "buffer subrange exceeds range");+ Self {+ buffer: self.buffer,+ range: self.range.start + range.start..self.range.start + range.end,+ }+ }}pub trait AsBufferRangeRef {@@ -98,6 +109,17 @@ impl<'a, B: Buffer> BufferRangeMut<'a, B> {pub fn range(&self) -> Range<usize> {self.range.clone()}++ pub(crate) fn subrange(+ self,+ range: Range<usize>,+ ) -> Self {+ assert!(range.end <= self.range.len(), "buffer subrange exceeds range");+ Self {+ buffer: self.buffer,+ range: self.range.start + range.start..self.range.start + range.end,+ }+ }}pub trait AsBufferRangeMut: AsBufferRangeRef {diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rsindex f636b18f4..7d6f9dd25 100644--- a/crates/backend-uzu/src/backends/common/command_buffer.rs+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs@@ -1,6 +1,6 @@-use std::{ops::Range, time::Duration};+use std::time::Duration;-use super::Backend;+use super::{Backend, BufferRangeMut, BufferRangeRef};pub trait CommandBuffer {type Backend: Backend<CommandBuffer = Self>;@@ -72,16 +72,13 @@ pub trait CommandBufferEncoding {fn encode_copy(&mut self,- src: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer,- src_range: Range<usize>,- dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer,- dst_range: Range<usize>,+ src: BufferRangeRef<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,+ dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,);fn encode_fill(&mut self,- dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer,- range: Range<usize>,+ dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,value: u8,);diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rsindex f92080244..25018ebd9 100644--- a/crates/backend-uzu/src/backends/common/encoder.rs+++ b/crates/backend-uzu/src/backends/common/encoder.rs@@ -1,12 +1,35 @@-use std::{ops::Range, time::Duration};+use std::{+ ops::{Bound, Range, RangeBounds},+ time::Duration,+};use crate::backends::common::{- AccessFlags, Allocation, AllocationPool, AllocationType, Backend, BufferGpuAddressRangeExt, CommandBuffer,- CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context,+ AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend,+ BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,+ CommandBufferInitial, CommandBufferPending, Context,hazard_tracker::{Access, HazardTracker},};+fn resolve_copy_range(+ range: impl RangeBounds<usize>,+ allocation_len: usize,+ label: &str,+) -> Range<usize> {+ let start = match range.start_bound() {+ Bound::Included(&value) => value,+ Bound::Excluded(&value) => value.checked_add(1).expect("copy range start overflow"),+ Bound::Unbounded => 0,+ };+ let end = match range.end_bound() {+ Bound::Included(&value) => value.checked_add(1).expect("copy range end overflow"),+ Bound::Excluded(&value) => value,+ Bound::Unbounded => allocation_len,+ };+ assert!(start <= end, "{label} copy range start exceeds end");+ assert!(end <= allocation_len, "{label} copy range exceeds allocation");+ start..end+}+pub struct Encoder<'encoding, B: Backend> {context: &'encoding B::Context,command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,@@ -58,37 +81,45 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {pub fn encode_copy(&mut self,- src: &B::DenseBuffer,- src_range: Range<usize>,- dst: &mut B::DenseBuffer,- dst_range: Range<usize>,+ src: &Allocation<B>,+ src_range: impl RangeBounds<usize>,+ dst: &mut Allocation<B>,+ dst_range: impl RangeBounds<usize>,) {- let size = src_range.end - src_range.start;- debug_assert_eq!(size, dst_range.end - dst_range.start);+ let src_buffer_range = src.as_buffer_range_ref();+ let dst_buffer_range = dst.as_buffer_range_mut();+ let src_range = resolve_copy_range(src_range, src_buffer_range.range().len(), "source");+ let dst_range = resolve_copy_range(dst_range, dst_buffer_range.range().len(), "destination");+ let byte_len = src_range.len();+ assert_eq!(byte_len, dst_range.len(), "copy range lengths must match");+ assert!(byte_len > 0, "zero-sized copies are not allowed");+ let src_buffer_range = src_buffer_range.subrange(src_range);+ let dst_buffer_range = dst_buffer_range.subrange(dst_range);self.access(&[Access {- range: src.gpu_address_subrange(src_range.clone()),+ range: src_buffer_range.buffer().gpu_address_subrange(src_buffer_range.range()),flags: AccessFlags::copy_read(),},Access {- range: dst.gpu_address_subrange(dst_range.clone()),+ range: dst_buffer_range.buffer().gpu_address_subrange(dst_buffer_range.range()),flags: AccessFlags::copy_write(),},]);- self.command_buffer.encode_copy(src, src_range, dst, dst_range);+ self.command_buffer.encode_copy(src_buffer_range, dst_buffer_range);}pub fn encode_fill(&mut self,- dst: &mut B::DenseBuffer,- range: Range<usize>,+ dst: &mut Allocation<B>,value: u8,) {+ let dst_buffer_range = dst.as_buffer_range_mut();+ assert!(!dst_buffer_range.range().is_empty(), "zero-sized fills are not allowed");self.access(&[Access {- range: dst.gpu_address_subrange(range.clone()),+ range: dst_buffer_range.buffer().gpu_address_subrange(dst_buffer_range.range()),flags: AccessFlags::copy_write(),}]);- self.command_buffer.encode_fill(dst, range, value);+ self.command_buffer.encode_fill(dst_buffer_range, value);}pub fn access(@@ -127,6 +158,10 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {&mut self.command_buffer}+ pub fn context(&self) -> &'encoding B::Context {+ self.context+ }+pub fn end_encoding(self) -> Executable<B> {Executable {command_buffer: self.command_buffer.end_encoding(),diff --git a/crates/backend-uzu/src/backends/common/kernel/attention.rs b/crates/backend-uzu/src/backends/common/kernel/attention.rsindex c56663580..5fcf577c1 100644--- a/crates/backend-uzu/src/backends/common/kernel/attention.rs+++ b/crates/backend-uzu/src/backends/common/kernel/attention.rs@@ -6,7 +6,7 @@ use std::{use crate::{DataType,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,gpu_types::{AttnParams, ring::RingParams},kernel::AttentionGemmKernel,},@@ -15,12 +15,12 @@ use crate::{const BQ: usize = 32;pub struct AttentionGemmArguments<'a, B: Backend> {- pub queries_buffer: &'a B::DenseBuffer,- pub keys_buffer: &'a B::DenseBuffer,- pub values_buffer: &'a B::DenseBuffer,- pub output_buffer: &'a mut B::DenseBuffer,- pub trie_buffer: Option<&'a B::DenseBuffer>,- pub sinks_buffer: Option<&'a B::DenseBuffer>,+ pub queries: &'a Allocation<B>,+ pub keys: &'a Allocation<B>,+ pub values: &'a Allocation<B>,+ pub output: &'a mut Allocation<B>,+ pub trie: Option<&'a Allocation<B>>,+ pub sinks: Option<&'a Allocation<B>>,pub num_heads: usize,pub num_groups: usize,pub suffix_length: usize, // qL@@ -58,7 +58,6 @@ impl<B: Backend> AttentionGemmBlock<B> {pub fn encode(&self,- context: &B::Context,encoder: &mut Encoder<B>,args: AttentionGemmArguments<B>,) -> Result<(), B::Error> {@@ -72,9 +71,9 @@ impl<B: Backend> AttentionGemmBlock<B> {let align_k = (args.sequence_length % bk) == 0;let is_kv_cache_ring = args.ring_params.is_some();let is_causal = args.is_causal;- let is_trie = args.trie_buffer.is_some();+ let is_trie = args.trie.is_some();let is_sliding_window = args.sliding_window_size.is_some();- let has_sinks = args.sinks_buffer.is_some();+ let has_sinks = args.sinks.is_some();let key = KernelKey {bk,head_dim,@@ -92,7 +91,7 @@ impl<B: Backend> AttentionGemmBlock<B> {Entry::Occupied(entry) => entry.into_mut(),Entry::Vacant(entry) => {let kernel = <B::Kernels as Kernels>::AttentionGemmKernel::new(- context,+ encoder.context(),self.data_type,bk as u32,head_dim as u32,@@ -137,15 +136,15 @@ impl<B: Backend> AttentionGemmBlock<B> {};kernel.encode(- args.queries_buffer,- args.keys_buffer,- args.values_buffer,- args.output_buffer,+ args.queries,+ args.keys,+ args.values,+ args.output,params,args.ring_params,- args.trie_buffer,+ args.trie,args.sliding_window_size.map(|s| s as u32),- args.sinks_buffer,+ args.sinks,args.num_heads as u32,args.suffix_length as u32,encoder,diff --git a/crates/backend-uzu/src/backends/common/kernel/audio.rs b/crates/backend-uzu/src/backends/common/kernel/audio.rsindex f7be333b6..dc4e3aa63 100644--- a/crates/backend-uzu/src/backends/common/kernel/audio.rs+++ b/crates/backend-uzu/src/backends/common/kernel/audio.rs@@ -7,9 +7,9 @@ pub enum AudioPadMode {}pub struct AudioFsqDecodeArguments<'a, B: Backend> {- pub tokens: &'a B::Buffer,- pub lengths: &'a B::Buffer,- pub output: &'a B::Buffer,+ pub tokens: &'a B::DenseBuffer,+ pub lengths: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,pub batch_size: usize,pub num_groups: usize,pub seq_len: usize,@@ -18,9 +18,9 @@ pub struct AudioFsqDecodeArguments<'a, B: Backend> {}pub struct AudioFsqEncodeArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub tokens: &'a B::Buffer,- pub lengths: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub tokens: &'a B::DenseBuffer,+ pub lengths: &'a B::DenseBuffer,pub batch_size: usize,pub num_groups: usize,pub seq_len: usize,@@ -31,11 +31,11 @@ pub struct AudioFsqEncodeArguments<'a, B: Backend> {}pub struct AudioConv1dArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub weight: &'a B::Buffer,- pub bias: &'a B::Buffer,- pub output: &'a B::Buffer,- pub lengths: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub weight: &'a B::DenseBuffer,+ pub bias: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,+ pub lengths: &'a B::DenseBuffer,pub batch_size: usize,pub cin: usize,pub cout: usize,@@ -49,11 +49,11 @@ pub struct AudioConv1dArguments<'a, B: Backend> {}pub struct AudioCausalConv1dArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub weight: &'a B::Buffer,- pub bias: &'a B::Buffer,- pub output: &'a B::Buffer,- pub lengths: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub weight: &'a B::DenseBuffer,+ pub bias: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,+ pub lengths: &'a B::DenseBuffer,pub batch_size: usize,pub cin: usize,pub cout: usize,@@ -63,11 +63,11 @@ pub struct AudioCausalConv1dArguments<'a, B: Backend> {}pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub weight: &'a B::Buffer,- pub bias: &'a B::Buffer,- pub output: &'a B::Buffer,- pub lengths: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub weight: &'a B::DenseBuffer,+ pub bias: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,+ pub lengths: &'a B::DenseBuffer,pub batch_size: usize,pub cin: usize,pub cout: usize,@@ -78,9 +78,9 @@ pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {}pub struct AudioHalfSnakeArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub alpha: &'a B::Buffer,- pub output: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub alpha: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,pub batch_size: usize,pub channels: usize,pub seq_len: usize,@@ -90,28 +90,28 @@ pub struct AudioHalfSnakeArguments<'a, B: Backend> {}pub struct AudioElementwiseArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub output: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,pub n: usize,}pub struct AudioAddArguments<'a, B: Backend> {- pub a: &'a B::Buffer,- pub b: &'a B::Buffer,- pub output: &'a B::Buffer,+ pub a: &'a B::DenseBuffer,+ pub b: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,pub n: usize,}pub struct AudioScaleArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub output: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,pub n: usize,pub scale: f32,}pub struct AudioClampArguments<'a, B: Backend> {- pub input: &'a B::Buffer,- pub output: &'a B::Buffer,+ pub input: &'a B::DenseBuffer,+ pub output: &'a B::DenseBuffer,pub n: usize,pub min_value: f32,pub max_value: f32,@@ -121,67 +121,67 @@ pub trait AudioKernelRuntime<B: Backend> {fn encode_fsq_decode(&self,encoder: &mut Encoder<B>,- arguments: AudioFsqDecodeArguments<'_, B>,+ arguments: AudioFsqDecodeArguments<B>,) -> Result<(), B::Error>;fn encode_fsq_encode(&self,encoder: &mut Encoder<B>,- arguments: AudioFsqEncodeArguments<'_, B>,+ arguments: AudioFsqEncodeArguments<B>,) -> Result<(), B::Error>;fn encode_conv1d(&self,encoder: &mut Encoder<B>,- arguments: AudioConv1dArguments<'_, B>,+ arguments: AudioConv1dArguments<B>,) -> Result<(), B::Error>;fn encode_causal_conv1d(&self,encoder: &mut Encoder<B>,- arguments: AudioCausalConv1dArguments<'_, B>,+ arguments: AudioCausalConv1dArguments<B>,) -> Result<(), B::Error>;fn encode_causal_conv_transpose1d(&self,encoder: &mut Encoder<B>,- arguments: AudioCausalConvTranspose1dArguments<'_, B>,+ arguments: AudioCausalConvTranspose1dArguments<B>,) -> Result<(), B::Error>;fn encode_half_snake(&self,encoder: &mut Encoder<B>,- arguments: AudioHalfSnakeArguments<'_, B>,+ arguments: AudioHalfSnakeArguments<B>,) -> Result<(), B::Error>;fn encode_leaky_relu(&self,encoder: &mut Encoder<B>,- arguments: AudioElementwiseArguments<'_, B>,+ arguments: AudioElementwiseArguments<B>,negative_slope: f32,) -> Result<(), B::Error>;fn encode_tanh(&self,encoder: &mut Encoder<B>,- arguments: AudioElementwiseArguments<'_, B>,+ arguments: AudioElementwiseArguments<B>,) -> Result<(), B::Error>;fn encode_add(&self,encoder: &mut Encoder<B>,- arguments: AudioAddArguments<'_, B>,+ arguments: AudioAddArguments<B>,) -> Result<(), B::Error>;fn encode_scale(&self,encoder: &mut Encoder<B>,- arguments: AudioScaleArguments<'_, B>,+ arguments: AudioScaleArguments<B>,) -> Result<(), B::Error>;fn encode_clamp(&self,encoder: &mut Encoder<B>,- arguments: AudioClampArguments<'_, B>,+ arguments: AudioClampArguments<B>,) -> Result<(), B::Error>;}diff --git a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rsindex 0c28fc409..6adb42def 100644--- a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs@@ -1,16 +1,16 @@-use std::{cell::RefCell, mem::size_of, ops::DerefMut, rc::Rc};+use std::mem::size_of;use thiserror::Error;use crate::{DataType,- backends::common::{Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},+ backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},};-pub struct KVLayerData<B: Backend> {- pub key_buffer: Rc<RefCell<B::DenseBuffer>>,+pub struct KVLayerData<'a, B: Backend> {+ pub key_allocation: &'a mut Allocation<B>,pub key_shape: [usize; 3],- pub value_buffer: Rc<RefCell<B::DenseBuffer>>,+ pub value_allocation: &'a mut Allocation<B>,pub value_shape: [usize; 3],}@@ -55,7 +55,7 @@ impl<B: Backend> KVCacheUpdate<B> {/// Encode the KV cache update operation using a provided command bufferpub fn encode(&self,- in_place_data: &[KVLayerData<B>],+ in_place_data: &mut [KVLayerData<B>],source_indices: &[usize],destination_indices: &[usize],encoder: &mut Encoder<B>,@@ -80,8 +80,8 @@ impl<B: Backend> KVCacheUpdate<B> {// non-inline is not supported yet (and is broken anyways due to a data race)for swaps_chunk in swaps.chunks(max_inline_swaps) {self.kernel.encode(- layer_data.key_buffer.borrow_mut().deref_mut(),- layer_data.value_buffer.borrow_mut().deref_mut(),+ &mut *layer_data.key_allocation,+ &mut *layer_data.value_allocation,swaps_chunk,swaps_chunk.len() as u32,num_heads as u32,diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rsindex 6e1008853..e8ef8c962 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul.rs@@ -2,7 +2,7 @@ use thiserror::Error;use crate::{DataType,- backends::common::{Backend, Encoder, kernel::ManualKernels},+ backends::common::{Allocation, Backend, Encoder, kernel::ManualKernels},};#[derive(Debug, Error)]@@ -13,29 +13,27 @@ pub enum MatmulError<B: Backend> {BackendError(#[source] B::Error),}-#[derive(Debug)]pub enum MatmulArgumentC<'a, B: Backend> {None,/// Accumulate: [M, N]Accumulate,/// Bias: [N] (broadcasted across M/batch)- Bias(&'a B::DenseBuffer),+ Bias(&'a Allocation<B>),}// D = ab_scale * (A @ B.T) + C-#[derive(Debug)]pub struct MatmulArguments<'a, B: Backend> {/// A: [M, K]- pub a: &'a B::DenseBuffer,- pub a_offset: u64,+ pub a: &'a Allocation<B>,+ pub a_offset: usize,/// B: [N, K]- pub b: &'a B::DenseBuffer,+ pub b: &'a Allocation<B>,/// AB scale: also known as alphapub ab_scale: f32,/// C: behavior depends on enum variantpub c: MatmulArgumentC<'a, B>,/// D: [M, N]- pub d: &'a mut B::DenseBuffer,+ pub d: &'a mut Allocation<B>,/// M dimension: usually batch/number of tokens (rows of A, rows of C)pub batch_dim: u32,/// K dimension: usually input_dim/reduction dimension (cols of A, rows of B)@@ -54,7 +52,6 @@ pub trait MatmulKernel: Sized {fn encode(&mut self,- context: &<Self::Backend as Backend>::Context,arguments: MatmulArguments<Self::Backend>,encoder: &mut Encoder<Self::Backend>,);diff --git a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rsindex 1c4784f9e..8676b3e1e 100644--- a/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -1,7 +1,7 @@use crate::{DataType,backends::common::{- ActivationConfig, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,+ ActivationConfig, Allocation, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,},};@@ -9,7 +9,7 @@ pub struct MlpGateActMulEncodable<B: Backend> {kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,activation: ActivationConfig,hidden_dim: usize,- hadamard_factors: Option<B::DenseBuffer>,+ hadamard_factors: Option<Allocation<B>>,}impl<B: Backend> MlpGateActMulEncodable<B> {@@ -18,7 +18,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {data_type: DataType,activation: ActivationConfig,hidden_dim: usize,- hadamard_factors: Option<B::DenseBuffer>,+ hadamard_factors: Option<Allocation<B>>,) -> Result<Self, B::Error> {let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type, hadamard_factors.is_some())?;Ok(Self {@@ -32,8 +32,8 @@ impl<B: Backend> MlpGateActMulEncodable<B> {pub fn encode(&self,encoder: &mut Encoder<B>,- fused_up: &B::DenseBuffer,- hidden: &mut B::DenseBuffer,+ fused_up: &Allocation<B>,+ hidden: &mut Allocation<B>,m: i32,) -> Result<(), B::Error> {if self.activation.act_type() == ActivationType::IDENTITY {diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rsindex fe30a1349..30ce0effd 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_single.rs@@ -1,9 +1,8 @@-use std::ops::{Deref, DerefMut};-use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{MoeExpertsDecodeSinglePassAKernel, MoeExpertsDecodeSinglePassBKernel},},};@@ -11,26 +10,21 @@ use crate::{static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];/// Arguments for single-token MoE decode (T=1 optimized path)-#[derive(Debug)]pub struct MoeExpertsSingleDecodeArguments<'a, B: Backend> {/// Input activation [d_model]- pub x: &'a B::DenseBuffer,+ pub x: &'a Allocation<B>,/// Top-K expert indices from router [K]- pub topk_ids: &'a B::DenseBuffer,+ pub topk_ids: &'a Allocation<B>,/// Top-K probabilities from router [K]- pub topk_probs: &'a B::DenseBuffer,+ pub topk_probs: &'a Allocation<B>,/// Up/gate projection weights [E, 2*d_ff, d_model]- pub w13_all: &'a B::DenseBuffer,+ pub w13_all: &'a Allocation<B>,/// Down projection weights [E, d_model, d_ff]- pub w2_all: &'a B::DenseBuffer,+ pub w2_all: &'a Allocation<B>,/// Up/gate biases [E, 2*d_ff]- pub up_biases: &'a B::DenseBuffer,+ pub up_biases: &'a Allocation<B>,/// Down biases [E, d_model]- pub down_biases: &'a B::DenseBuffer,- /// Hidden buffer [K, d_ff] - intermediate storage (f32)- pub hidden: &'a mut B::DenseBuffer,- /// Final output [d_model]- pub y: &'a mut B::DenseBuffer,+ pub down_biases: &'a Allocation<B>,/// Model dimensionpub d_model: usize,/// FFN hidden dimension@@ -85,14 +79,14 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {pub fn encode(&self,encoder: &mut Encoder<B>,- mut args: MoeExpertsSingleDecodeArguments<B>,- ) {- if args.k == 0 {- return;- }+ args: MoeExpertsSingleDecodeArguments<B>,+ ) -> Result<Allocation<B>, B::Error> {+ assert!(args.k > 0, "MoE single decode requires at least one active expert");let gate_idx = args.gating_code.min(3) as usize;let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.k, args.d_ff], DataType::F32))?;+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.d_model], args.data_type))?;// Pass A: x @ W13[expert] -> hiddenlet kernel = &self.pass_a[gate_idx][dtype_idx];@@ -101,7 +95,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {args.topk_ids,args.w13_all,args.up_biases,- args.hidden.deref_mut(),+ &mut hidden,args.d_model as u32,args.d_ff as u32,args.k as u32,@@ -116,16 +110,17 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {// Pass B: 8 simdgroups (256 threads), outputs final y directlylet kernel = &self.pass_b[dtype_idx];kernel.encode(- args.hidden.deref(),+ &hidden,args.topk_ids,args.topk_probs,args.w2_all,args.down_biases,- args.y,+ &mut output,args.d_model as u32,args.d_ff as u32,args.k as u32,encoder,);+ Ok(output)}}diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rsindex 0749d385e..8fc689c81 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs@@ -1,13 +1,12 @@-use std::ops::{Deref, DerefMut};-use super::{MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,};use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},},};@@ -48,11 +47,10 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {pub fn encode(&self,encoder: &mut Encoder<B>,- mut args: MoeExpertsTwoPassArguments<B>,- ) {- if args.total_rows == 0 {- return;- }+ args: MoeExpertsTwoPassArguments<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;// pass a tileconst BLOCK_M: u32 = 4;@@ -61,7 +59,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {encoder,MoePassATileCountsArguments {expert_offsets: args.expert_offsets,- tile_counts: args.tile_counts.deref_mut(),+ tile_counts: &mut *args.tile_counts,e: args.e,h_blocks,},@@ -69,9 +67,9 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {self.pass_a_tile.encode_scan(encoder,MoePassATileScanArguments {- tile_counts: args.tile_counts,- tile_offsets: args.tile_offsets.deref_mut(),- total_tiles: args.total_tiles.deref_mut(),+ tile_counts: &*args.tile_counts,+ tile_offsets: &mut *args.tile_offsets,+ total_tiles: &mut *args.total_tiles,e: args.e,},);@@ -79,7 +77,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {encoder,MoePassARowMapArguments {expert_offsets: args.expert_offsets,- row_expert_map: args.row_expert_map.deref_mut(),+ row_expert_map: &mut *args.row_expert_map,total_rows: args.total_rows,e: args.e,},@@ -88,9 +86,9 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {encoder,MoePassATileBuildArguments {expert_offsets: args.expert_offsets,- tile_offsets: args.tile_offsets,- row_expert_map: args.row_expert_map,- tile_map: args.tile_map.deref_mut(),+ tile_offsets: &*args.tile_offsets,+ row_expert_map: &*args.row_expert_map,+ tile_map: &mut *args.tile_map,total_rows: args.total_rows,h_blocks,},@@ -98,8 +96,8 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {self.pass_a_tile.encode_dispatch_args(encoder,MoePassATileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args.deref_mut(),+ total_tiles: &*args.total_tiles,+ dispatch_args: &mut *args.dispatch_args,num_tiles_y: 1,},);@@ -110,10 +108,10 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {// pass alet pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];pass_a_kernel.encode(- args.x_perm_buffer,+ args.x_perm,args.expert_offsets,args.w13_all,- args.hidden_buffer.deref_mut(),+ &mut hidden,args.up_biases,args.d_model as u32,args.d_ff as u32,@@ -123,24 +121,25 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {args.up_clip_min,args.up_clip_max,args.silu_alpha,- args.tile_map.deref(),- args.dispatch_args.deref(),+ &*args.tile_map,+ &*args.dispatch_args,encoder,);// pass blet pass_b_kernel = &self.fused_down[dtype_idx];pass_b_kernel.encode(- args.hidden_buffer.deref(),- args.row_expert_map.deref(),+ &hidden,+ &*args.row_expert_map,args.w2_all,args.down_biases,- args.output_buffer.deref_mut(),+ &mut output,args.total_rows as u32,args.d_model as u32,args.d_ff as u32,args.e as u32,encoder,);+ Ok(output)}}diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rsindex 25a781aa2..c4f65ef6e 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs@@ -1,12 +1,11 @@-use std::ops::{Deref, DerefMut};-use super::{MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,};use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},},};@@ -49,29 +48,26 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {pub fn encode(&self,encoder: &mut Encoder<B>,- mut args: MoeExpertsTwoPassArguments<B>,- ) {- if args.total_rows == 0 {- return;- }-- let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();- encoder.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);+ args: MoeExpertsTwoPassArguments<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_ff], DataType::F32))?;+ let mut output = encoder.allocate_scratch(size_for_shape(&[args.total_rows, args.d_model], args.data_type))?;+ encoder.encode_fill(&mut hidden, 0);self.tile_map.encode_counts(encoder,MoeTileCountsArguments {- offsets_buffer: args.expert_offsets,- tile_counts_buffer: args.tile_counts.deref_mut(),+ offsets: args.expert_offsets,+ tile_counts: &mut *args.tile_counts,e: args.e,},);self.tile_map.encode_scan(encoder,MoeTileScanArguments {- tile_counts_buffer: args.tile_counts,- tile_offsets_buffer: args.tile_offsets.deref_mut(),- total_tiles_buffer: args.total_tiles.deref_mut(),+ tile_counts: &*args.tile_counts,+ tile_offsets: &mut *args.tile_offsets,+ total_tiles: &mut *args.total_tiles,e: args.e,},);@@ -79,9 +75,9 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {encoder,MoeTileMapBuildArguments {expert_offsets: args.expert_offsets,- tile_offsets: args.tile_offsets,- tile_counts: args.tile_counts,- tile_map: args.tile_map.deref_mut(),+ tile_offsets: &*args.tile_offsets,+ tile_counts: &*args.tile_counts,+ tile_map: &mut *args.tile_map,e: args.e,},);@@ -102,8 +98,8 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {self.tile_map.encode_dispatch_args(encoder,MoeTileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args.deref_mut(),+ total_tiles: &*args.total_tiles,+ dispatch_args: &mut *args.dispatch_args,num_tiles_x: n_tiles_ff,},);@@ -113,11 +109,11 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];kernel_pass_a.encode(- args.x_perm_buffer,+ args.x_perm,args.expert_offsets,args.w13_all,args.up_biases,- args.hidden_buffer.deref_mut(),+ &mut hidden,args.d_model as u32,args.d_ff as u32,args.e as u32,@@ -126,51 +122,49 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args.up_clip_min,args.up_clip_max,args.silu_alpha,- args.tile_map.deref(),- args.dispatch_args.deref(),+ &*args.tile_map,+ &*args.dispatch_args,encoder,);let dispatch_args = MoeTileDispatchArguments {- total_tiles: args.total_tiles,- dispatch_args: args.dispatch_args.deref_mut(),+ total_tiles: &*args.total_tiles,+ dispatch_args: &mut *args.dispatch_args,num_tiles_x: n_tiles_model,};self.tile_map.encode_dispatch_args(encoder, dispatch_args);let kernel_pass_b = &self.pass_b_indirect[dtype_idx];kernel_pass_b.encode(- args.hidden_buffer.deref(),+ &hidden,args.expert_offsets,args.w2_all,args.down_biases,- args.output_buffer.deref_mut(),+ &mut output,args.d_model as u32,args.d_ff as u32,args.e as u32,- args.tile_map.deref(),- args.dispatch_args.deref(),+ &*args.tile_map,+ &*args.dispatch_args,encoder,);+ Ok(output)}}-#[derive(Debug)]pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {- pub x_perm_buffer: &'a B::DenseBuffer,- pub expert_offsets: &'a B::DenseBuffer,- pub row_expert_map: &'a mut B::DenseBuffer,- pub hidden_buffer: &'a mut B::DenseBuffer,- pub output_buffer: &'a mut B::DenseBuffer,- pub w13_all: &'a B::DenseBuffer,- pub w2_all: &'a B::DenseBuffer,- pub up_biases: &'a B::DenseBuffer,- pub down_biases: &'a B::DenseBuffer,- pub tile_counts: &'a mut B::DenseBuffer,- pub tile_offsets: &'a mut B::DenseBuffer,- pub tile_map: &'a mut B::DenseBuffer,- pub total_tiles: &'a mut B::DenseBuffer,- pub dispatch_args: &'a mut B::DenseBuffer,+ pub x_perm: &'a Allocation<B>,+ pub expert_offsets: &'a Allocation<B>,+ pub row_expert_map: &'a mut Allocation<B>,+ pub w13_all: &'a Allocation<B>,+ pub w2_all: &'a Allocation<B>,+ pub up_biases: &'a Allocation<B>,+ pub down_biases: &'a Allocation<B>,+ pub tile_counts: &'a mut Allocation<B>,+ pub tile_offsets: &'a mut Allocation<B>,+ pub tile_map: &'a mut Allocation<B>,+ pub total_tiles: &'a mut Allocation<B>,+ pub dispatch_args: &'a mut Allocation<B>,pub total_rows: usize,pub d_model: usize,pub d_ff: usize,diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rsindex fc6887896..9c6040b37 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/gather.rs@@ -1,17 +1,16 @@use crate::{DataType,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},},};-#[derive(Debug)]pub struct MoeGatherArguments<'a, B: Backend> {- pub x_buffer: &'a B::DenseBuffer,- pub bucketed_ids_buffer: &'a B::DenseBuffer,- pub x_perm_buffer: &'a mut B::DenseBuffer,- pub sumk_buffer: &'a B::DenseBuffer,+ pub x: &'a Allocation<B>,+ pub bucketed_ids: &'a Allocation<B>,+ pub x_perm: &'a mut Allocation<B>,+ pub sumk: &'a Allocation<B>,pub t: usize,pub k: usize,pub d_model: usize,@@ -40,30 +39,30 @@ impl<B: Backend> MoeGatherKernels<B> {) {match dtype {DataType::F32 => self.f32.encode(- args.x_buffer,- args.bucketed_ids_buffer,- args.x_perm_buffer,- args.sumk_buffer,+ args.x,+ args.bucketed_ids,+ args.x_perm,+ args.sumk,args.d_model as u32,args.t as u32,args.k as u32,encoder,),DataType::F16 => self.f16.encode(- args.x_buffer,- args.bucketed_ids_buffer,- args.x_perm_buffer,- args.sumk_buffer,+ args.x,+ args.bucketed_ids,+ args.x_perm,+ args.sumk,args.d_model as u32,args.t as u32,args.k as u32,encoder,),DataType::BF16 => self.bf16.encode(- args.x_buffer,- args.bucketed_ids_buffer,- args.x_perm_buffer,- args.sumk_buffer,+ args.x,+ args.bucketed_ids,+ args.x_perm,+ args.sumk,args.d_model as u32,args.t as u32,args.k as u32,diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rsindex 3bb9871fc..6bff799cc 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_map.rs@@ -1,5 +1,5 @@use crate::backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},};@@ -25,7 +25,7 @@ impl<B: Backend> MoeTileMapKernels<B> {encoder: &mut Encoder<B>,args: MoeTileCountsArguments<B>,) {- self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);+ self.counts.encode(args.offsets, args.tile_counts, args.e as u32, encoder);}pub fn encode_scan(@@ -33,13 +33,7 @@ impl<B: Backend> MoeTileMapKernels<B> {encoder: &mut Encoder<B>,args: MoeTileScanArguments<B>,) {- self.scan.encode(- args.tile_counts_buffer,- args.tile_offsets_buffer,- args.total_tiles_buffer,- args.e as u32,- encoder,- );+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);}pub fn encode_build_map(@@ -67,30 +61,28 @@ impl<B: Backend> MoeTileMapKernels<B> {}pub struct MoeTileCountsArguments<'a, B: Backend> {- pub offsets_buffer: &'a B::DenseBuffer, // [E+1]- pub tile_counts_buffer: &'a mut B::DenseBuffer, // [E]+ pub offsets: &'a Allocation<B>, // [E+1]+ pub tile_counts: &'a mut Allocation<B>, // [E]pub e: usize,}pub struct MoeTileScanArguments<'a, B: Backend> {- pub tile_counts_buffer: &'a B::DenseBuffer, // [E]- pub tile_offsets_buffer: &'a mut B::DenseBuffer, // [E+1]- pub total_tiles_buffer: &'a mut B::DenseBuffer, // [>=2]+ pub tile_counts: &'a Allocation<B>, // [E]+ pub tile_offsets: &'a mut Allocation<B>, // [E+1]+ pub total_tiles: &'a mut Allocation<B>, // [>=2]pub e: usize,}-#[derive(Debug)]pub struct MoeTileMapBuildArguments<'a, B: Backend> {- pub expert_offsets: &'a B::DenseBuffer, // [E+1]- pub tile_offsets: &'a B::DenseBuffer, // [E+1]- pub tile_counts: &'a B::DenseBuffer, // [E]- pub tile_map: &'a mut B::DenseBuffer, // [total_tiles * 3]+ pub expert_offsets: &'a Allocation<B>, // [E+1]+ pub tile_offsets: &'a Allocation<B>, // [E+1]+ pub tile_counts: &'a Allocation<B>, // [E]+ pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]pub e: usize,}-#[derive(Debug)]pub struct MoeTileDispatchArguments<'a, B: Backend> {- pub total_tiles: &'a B::DenseBuffer, // [>=1]- pub dispatch_args: &'a mut B::DenseBuffer, // [3]- pub num_tiles_x: u32, // x dimension for indirect dispatch+ pub total_tiles: &'a Allocation<B>, // [>=1]+ pub dispatch_args: &'a mut Allocation<B>, // [3]+ pub num_tiles_x: u32, // x dimension for indirect dispatch}diff --git a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rsindex 69ee10cf3..cc3f8674e 100644--- a/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs+++ b/crates/backend-uzu/src/backends/common/kernel/moe/tiles_pass_a.rs@@ -1,49 +1,44 @@use crate::backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,},};-#[derive(Debug)]pub struct MoePassATileCountsArguments<'a, B: Backend> {- pub expert_offsets: &'a B::DenseBuffer, // [E+1]- pub tile_counts: &'a mut B::DenseBuffer, // [E]+ pub expert_offsets: &'a Allocation<B>, // [E+1]+ pub tile_counts: &'a mut Allocation<B>, // [E]pub e: usize,pub h_blocks: u32,}-#[derive(Debug)]pub struct MoePassATileScanArguments<'a, B: Backend> {- pub tile_counts: &'a B::DenseBuffer, // [E]- pub tile_offsets: &'a mut B::DenseBuffer, // [E+1]- pub total_tiles: &'a mut B::DenseBuffer, // [>=1]+ pub tile_counts: &'a Allocation<B>, // [E]+ pub tile_offsets: &'a mut Allocation<B>, // [E+1]+ pub total_tiles: &'a mut Allocation<B>, // [>=1]pub e: usize,}-#[derive(Debug)]pub struct MoePassARowMapArguments<'a, B: Backend> {- pub expert_offsets: &'a B::DenseBuffer, // [E+1]- pub row_expert_map: &'a mut B::DenseBuffer, // [total_rows]+ pub expert_offsets: &'a Allocation<B>, // [E+1]+ pub row_expert_map: &'a mut Allocation<B>, // [total_rows]pub total_rows: usize,pub e: usize,}-#[derive(Debug)]pub struct MoePassATileBuildArguments<'a, B: Backend> {- pub expert_offsets: &'a B::DenseBuffer, // [E+1]- pub tile_offsets: &'a B::DenseBuffer, // [E+1]- pub row_expert_map: &'a B::DenseBuffer, // [total_rows]- pub tile_map: &'a mut B::DenseBuffer, // [total_tiles * 3]+ pub expert_offsets: &'a Allocation<B>, // [E+1]+ pub tile_offsets: &'a Allocation<B>, // [E+1]+ pub row_expert_map: &'a Allocation<B>, // [total_rows]+ pub tile_map: &'a mut Allocation<B>, // [total_tiles * 3]pub total_rows: usize,pub h_blocks: u32,}-#[derive(Debug)]pub struct MoePassATileDispatchArguments<'a, B: Backend> {- pub total_tiles: &'a B::DenseBuffer, // [>=1]- pub dispatch_args: &'a mut B::DenseBuffer, // [3]+ pub total_tiles: &'a Allocation<B>, // [>=1]+ pub dispatch_args: &'a mut Allocation<B>, // [3]pub num_tiles_y: u32,}diff --git a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rsindex 4320f290b..af80634fe 100644--- a/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/backend-uzu/src/backends/common/kernel/quant_matmul.rs@@ -1,7 +1,7 @@use crate::{DataType,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,gpu_types::QuantizationMode,kernel::{QuantizedMatmulQmmTransposedKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},},@@ -37,13 +37,13 @@ pub struct QuantizedMatmulConfiguration {}pub struct QuantizedMatmulArguments<'a, B: Backend> {- pub a_buffer: &'a B::DenseBuffer,+ pub a: &'a Allocation<B>,pub a_offset: usize,- pub b_buffer: &'a B::DenseBuffer,- pub scales_buffer: &'a B::DenseBuffer,- pub zero_points_or_biases_buffer: &'a B::DenseBuffer,- pub output_buffer: &'a mut B::DenseBuffer,- pub hadamard_factors: Option<&'a B::DenseBuffer>,+ pub b: &'a Allocation<B>,+ pub scales: &'a Allocation<B>,+ pub zero_points_or_biases: &'a Allocation<B>,+ pub output: &'a mut Allocation<B>,+ pub hadamard_factors: Option<&'a Allocation<B>>,pub batch_dim: usize,}@@ -62,7 +62,6 @@ enum MatrixVectorKernel<B: Backend> {type QmmKernel<B> = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel;-/// Big tile alone, or paired with a BM=8 small tile for `batch_dim < 48`.enum MatrixMatrixKernel<B: Backend> {Big(QmmKernel<B>),BigAndSmall {@@ -72,7 +71,6 @@ enum MatrixMatrixKernel<B: Backend> {}impl<B: Backend> MatrixMatrixKernel<B> {- /// `None` means fall back to `matrix_vector` (small `batch_dim`, no small tile available).fn pick(&self,batch_dim: usize,@@ -112,7 +110,6 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {let use_mlx_quant = matches!(configuration.quantization_type, QuantizedMatmulType::Mlx);let use_zero_points = !use_mlx_quant;- // Matrix-vectorlet matrix_vector = if configuration.output_dim % 8 == 0 && configuration.input_dim % 512 == 0 {MatrixVectorKernel::QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(@@ -144,8 +141,6 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {)};- // 64-tile (BM=BN=64): bf16 + output_dim%64==0; BK=64 if group_size≥64, BK=32 otherwise.- // 32-tile fallback otherwise.let aligned_n_64 = configuration.output_dim % 64 == 0;let aligned_n_32 = configuration.output_dim % 32 == 0;let is_bf16 = configuration.data_type == DataType::BF16;@@ -167,8 +162,8 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {bm,bk,bn,- 2u32, // WM- 2u32, // WN+ 2u32,+ 2u32,use_zero_points,use_mlx_quant,configuration.use_hadamard,@@ -176,23 +171,21 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {).map_err(QuantizedMatmulError::BackendError)?;- // Small-tile kernel (BM=8): batch_dim in [8, 48). Skipped for hadamard configs- // since the BM=8 tile doesn't support it; QmvFast handles those batch sizes instead.let matrix_matrix = if aligned_n_32 && !configuration.use_hadamard {let small = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(context,configuration.data_type,group_size,bits,- 8u32, // BM- 32u32, // BK- 32u32, // BN- 1u32, // WM- 1u32, // WN+ 8u32,+ 32u32,+ 32u32,+ 1u32,+ 1u32,use_zero_points,use_mlx_quant,- false, // use_hadamard: BM=8 tile does not support hadamard- true, // aligned_n: BN=32, always aligned for the small tile+ false,+ true,).map_err(QuantizedMatmulError::BackendError)?;MatrixMatrixKernel::BigAndSmall {@@ -216,45 +209,52 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {&self,encoder: &mut Encoder<B>,arguments: QuantizedMatmulArguments<B>,- ) -> Result<(), QuantizedMatmulError<B>> {+ ) {+ let QuantizedMatmulArguments {+ a,+ a_offset,+ b,+ scales,+ zero_points_or_biases,+ output,+ hadamard_factors,+ batch_dim,+ } = arguments;+let (zero_points, biases) = match self.quantization_type {- QuantizedMatmulType::ZeroPoint => (Some(arguments.zero_points_or_biases_buffer), None),- QuantizedMatmulType::Mlx => (None, Some(arguments.zero_points_or_biases_buffer)),+ QuantizedMatmulType::ZeroPoint => (Some(zero_points_or_biases), None),+ QuantizedMatmulType::Mlx => (None, Some(zero_points_or_biases)),};macro_rules! encode_kernel {- ($kernel:expr, $($hadamard:expr)?) => {+ ($kernel:expr $(, $hadamard:expr)?) => {$kernel.encode(- arguments.b_buffer,- arguments.scales_buffer,+ b,+ scales,zero_points,biases,- (arguments.a_buffer, arguments.a_offset),- arguments.output_buffer,+ (a, a_offset),+ output,$($hadamard,)?self.input_dim as u32,self.output_dim as u32,- arguments.batch_dim as u32,+ batch_dim as u32,encoder,)};}- if arguments.batch_dim >= 5 && self.output_dim > 1 {- if let Some(kernel) = self.matrix_matrix.pick(arguments.batch_dim) {- encode_kernel!(kernel, arguments.hadamard_factors);- return Ok(());+ if batch_dim >= 5 && self.output_dim > 1 {+ if let Some(kernel) = self.matrix_matrix.pick(batch_dim) {+ encode_kernel!(kernel, hadamard_factors);+ return;}}- // Matrix-vector fallback: small batch_dim, scalar output, or Big-tile-only- // configs that don't cover this batch_dim.match &self.matrix_vector {- MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k),+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, hadamard_factors),}-- Ok(())}}diff --git a/crates/backend-uzu/src/backends/common/kernel/sampling.rs b/crates/backend-uzu/src/backends/common/kernel/sampling.rsindex 9f0c24fc4..34e78135f 100644--- a/crates/backend-uzu/src/backends/common/kernel/sampling.rs+++ b/crates/backend-uzu/src/backends/common/kernel/sampling.rs@@ -1,15 +1,11 @@-use std::{- cell::RefCell,- mem::size_of,- ops::{Deref, DerefMut},-};+use std::mem::size_of;use thiserror::Error;use crate::{DataType,backends::common::{- Backend, Context, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,gpu_types::ArgmaxPair,kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,@@ -32,7 +28,6 @@ enum ArgmaxImplementation<B: Backend> {TwoPass {main_kernel: <B::Kernels as Kernels>::ArgmaxMainKernel,final_kernel: <B::Kernels as Kernels>::ArgmaxFinalKernel,- partial_results_buffer: RefCell<B::DenseBuffer>,},}@@ -44,8 +39,6 @@ pub struct SamplingKernel<B: Backend> {minp: <B::Kernels as Kernels>::MinPKernel,gumbel: <B::Kernels as Kernels>::GumbelKernel,argmax_implementation: ArgmaxImplementation<B>,- max_batch_size: usize,- max_vocab_size: usize,}#[derive(Debug, Error)]@@ -54,27 +47,23 @@ pub enum SamplingError<B: Backend> {BackendError(#[source] B::Error),#[error("Function not found: {0}")]FunctionNotFound(String),- #[error("Batch size {0} exceeds maximum {1}")]- BatchSizeExceeded(usize, usize),- #[error("Vocab size {0} exceeds maximum {1}")]- VocabSizeExceeded(usize, usize),+ #[error("batch size must be greater than 0")]+ EmptyBatch,+ #[error("vocab size must be greater than 0")]+ EmptyVocab,}impl<B: Backend> SamplingKernel<B> {pub fn new(context: &B::Context,data_type: DataType,- max_batch_size: usize,- max_vocab_size: usize,) -> Result<Self, SamplingError<B>> {- Self::new_with_strategy(context, data_type, max_batch_size, max_vocab_size, ArgmaxStrategy::TwoPass)+ Self::new_with_strategy(context, data_type, ArgmaxStrategy::TwoPass)}pub fn new_with_strategy(context: &B::Context,data_type: DataType,- max_batch_size: usize,- max_vocab_size: usize,argmax_strategy: ArgmaxStrategy,) -> Result<Self, SamplingError<B>> {let bitmask = <B::Kernels as Kernels>::BitmaskKernel::new(context, data_type, true)@@ -105,23 +94,9 @@ impl<B: Backend> SamplingKernel<B> {let final_kernel =<B::Kernels as Kernels>::ArgmaxFinalKernel::new(context).map_err(SamplingError::BackendError)?;- let block_size = 1024;- let grain_size = 4;-- let elements_per_group = block_size * grain_size;- let max_vocab_groups_per_batch = (max_vocab_size + elements_per_group - 1) / elements_per_group;- let max_partial_results = max_batch_size * max_vocab_groups_per_batch;-- let partial_results_buffer = RefCell::new(- context- .create_buffer(max_partial_results * size_of::<ArgmaxPair>())- .expect("Failed to create partial results buffer"),- );-ArgmaxImplementation::TwoPass {main_kernel,final_kernel,- partial_results_buffer,}},};@@ -134,36 +109,32 @@ impl<B: Backend> SamplingKernel<B> {minp,gumbel,argmax_implementation,- max_batch_size,- max_vocab_size,})}pub fn encode(&self,- mut logits_buffer: &mut B::DenseBuffer,- seeds_buffer: &B::DenseBuffer,- seeds_offset: usize,- bitmask_buffer: Option<&B::DenseBuffer>,- bitmask_offset: usize,- sampled_tokens_buffer: &mut B::DenseBuffer,+ logits: &mut Allocation<B>,+ seeds: &Allocation<B>,+ bitmask: Option<&Allocation<B>>,+ sampled_tokens: &mut Allocation<B>,sampling_method: SamplingMethod,batch_size: usize,vocab_size: usize,encoder: &mut Encoder<B>,) -> Result<(), SamplingError<B>> {- if batch_size > self.max_batch_size {- return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));+ if batch_size == 0 {+ return Err(SamplingError::EmptyBatch);}- if vocab_size > self.max_vocab_size {- return Err(SamplingError::VocabSizeExceeded(vocab_size, self.max_vocab_size));+ if vocab_size == 0 {+ return Err(SamplingError::EmptyVocab);}- if let Some(bitmask_buffer) = bitmask_buffer {+ if let Some(bitmask) = bitmask {self.bitmask.encode(- None::<&B::DenseBuffer>,- (bitmask_buffer, bitmask_offset),- logits_buffer.deref_mut(),+ None::<&Allocation<B>>,+ bitmask,+ &mut *logits,batch_size as u32,vocab_size as u32,encoder,@@ -182,8 +153,8 @@ impl<B: Backend> SamplingKernel<B> {&& processing_order == SamplingProcessingOrder::TemperatureThenFilters{self.temperature.encode(- None::<&B::DenseBuffer>,- logits_buffer.deref_mut(),+ None::<&Allocation<B>>,+ &mut *logits,batch_size as u32,vocab_size as u32,temperature,@@ -193,8 +164,8 @@ impl<B: Backend> SamplingKernel<B> {if let Some(top_k) = top_k {self.topk.encode(- None::<&B::DenseBuffer>,- logits_buffer.deref_mut(),+ None::<&Allocation<B>>,+ &mut *logits,batch_size as u32,vocab_size as u32,top_k,@@ -203,8 +174,8 @@ impl<B: Backend> SamplingKernel<B> {}if let Some(top_p) = top_p {self.topp.encode(- None::<&B::DenseBuffer>,- logits_buffer.deref_mut(),+ None::<&Allocation<B>>,+ &mut *logits,batch_size as u32,vocab_size as u32,top_p,@@ -213,8 +184,8 @@ impl<B: Backend> SamplingKernel<B> {}if let Some(min_p) = min_p {self.minp.encode(- None::<&B::DenseBuffer>,- logits_buffer.deref_mut(),+ None::<&Allocation<B>>,+ &mut *logits,batch_size as u32,vocab_size as u32,min_p,@@ -226,8 +197,8 @@ impl<B: Backend> SamplingKernel<B> {&& processing_order == SamplingProcessingOrder::FiltersThenTemperature{self.temperature.encode(- None::<&B::DenseBuffer>,- logits_buffer.deref_mut(),+ None::<&Allocation<B>>,+ &mut *logits,batch_size as u32,vocab_size as u32,temperature,@@ -236,10 +207,10 @@ impl<B: Backend> SamplingKernel<B> {}self.gumbel.encode(- None::<&B::DenseBuffer>,- seeds_buffer,- (seeds_offset / size_of::<u64>()) as u32,- logits_buffer.deref_mut(),+ None::<&Allocation<B>>,+ seeds,+ 0,+ &mut *logits,batch_size as u32,vocab_size as u32,encoder,@@ -250,33 +221,22 @@ impl<B: Backend> SamplingKernel<B> {ArgmaxImplementation::SinglePass {kernel,} => {- kernel.encode(- logits_buffer.deref(),- sampled_tokens_buffer,- batch_size as u32,- vocab_size as u32,- encoder,- );+ kernel.encode(&*logits, sampled_tokens, batch_size as u32, vocab_size as u32, encoder);},ArgmaxImplementation::TwoPass {main_kernel,final_kernel,- partial_results_buffer,} => {- main_kernel.encode(- logits_buffer.deref(),- partial_results_buffer.borrow_mut().deref_mut(),- batch_size as u32,- vocab_size as u32,- encoder,- );- final_kernel.encode(- partial_results_buffer.borrow().deref(),- sampled_tokens_buffer,- batch_size as u32,- vocab_size as u32,- encoder,- );+ let block_size = 1024;+ let grain_size = 4;+ let elements_per_group = block_size * grain_size;+ let vocab_groups_per_batch = vocab_size.div_ceil(elements_per_group);+ let partial_results_count = batch_size * vocab_groups_per_batch;+ let mut partial_results = encoder+ .allocate_scratch(partial_results_count * size_of::<ArgmaxPair>())+ .map_err(SamplingError::BackendError)?;+ main_kernel.encode(&*logits, &mut partial_results, batch_size as u32, vocab_size as u32, encoder);+ final_kernel.encode(&partial_results, sampled_tokens, batch_size as u32, vocab_size as u32, encoder);},}diff --git a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rsindex 64f473798..b3769ec4a 100644--- a/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs+++ b/crates/backend-uzu/src/backends/common/kernel/ssd_prefill.rs@@ -1,7 +1,7 @@use crate::{DataType,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},},};@@ -13,14 +13,14 @@ pub enum SSDPrefillMode {}pub struct SSDPrefillArguments<'a, B: Backend> {- pub x: &'a B::DenseBuffer,- pub dt: &'a B::DenseBuffer, // raw dt values- pub b: &'a B::DenseBuffer,- pub c: &'a B::DenseBuffer,- pub d: &'a B::DenseBuffer,- pub z: &'a B::DenseBuffer,- pub state: &'a mut B::DenseBuffer,- pub y: &'a mut B::DenseBuffer,+ pub x: &'a Allocation<B>,+ pub dt: &'a Allocation<B>,+ pub b: &'a Allocation<B>,+ pub c: &'a Allocation<B>,+ pub d: &'a Allocation<B>,+ pub z: &'a Allocation<B>,+ pub state: &'a mut Allocation<B>,+ pub y: &'a mut Allocation<B>,pub suffix_len: usize,pub group_size: u32,pub state_size: u32,diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rsindex 848846f45..8dc4adada 100644--- a/crates/backend-uzu/src/backends/common/mod.rs+++ b/crates/backend-uzu/src/backends/common/mod.rs@@ -1,4 +1,5 @@mod activation_config;+mod allocation_access;mod allocator;mod backend;mod buffer;@@ -12,6 +13,9 @@ mod hazard_tracker;pub mod kernel;pub use activation_config::ActivationConfig;+pub use allocation_access::{+ AllocationAccessError, allocation_as_bytes, allocation_as_bytes_mut, allocation_copy_from_slice, allocation_to_vec,+};pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};pub use backend::Backend;pub use buffer::{diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rsindex 7d2d8abcb..f2056e35e 100644--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs@@ -1,5 +1,4 @@use std::{- cell::UnsafeCell,pin::Pin,sync::{atomic::{AtomicU64, Ordering},@@ -12,8 +11,8 @@ use std::{use crate::{backends::{common::{- AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferInitial, CommandBufferPending,+ AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,},cpu::{Cpu, error::CpuError},},@@ -76,18 +75,16 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {fn encode_copy(&mut self,- src: &UnsafeCell<Pin<Box<[u8]>>>,- src_range: std::ops::Range<usize>,- dst: &mut UnsafeCell<Pin<Box<[u8]>>>,- dst_range: std::ops::Range<usize>,+ src: BufferRangeRef<'_, <Cpu as Backend>::DenseBuffer>,+ dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,) {+ let src_range = src.range();+ let dst_range = dst.range();let size = src_range.end - src_range.start;assert_eq!(size, dst_range.end - dst_range.start);- assert!(unsafe { &*src.get() }.len() >= src_range.end);- assert!(dst.get_mut().len() >= dst_range.end);- let src_ptr = SendPtr(unsafe { (&*src.get()).as_ptr().add(src_range.start) });- let dst_ptr = SendPtrMut(unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) });+ let src_ptr = SendPtr(unsafe { (&*src.buffer().get()).as_ptr().add(src_range.start) });+ let dst_ptr = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(dst_range.start) });self.push_command(move || unsafe {std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), size);});@@ -95,12 +92,12 @@ impl CommandBufferEncoding for CpuCommandBufferEncoding {fn encode_fill(&mut self,- dst: &mut UnsafeCell<Pin<Box<[u8]>>>,- range: std::ops::Range<usize>,+ dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,value: u8,) {+ let range = dst.range();let size = range.end - range.start;- let dst = SendPtrMut(dst.get_mut()[range].as_ptr() as *mut u8);+ let dst = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(range.start) });self.push_command(move || unsafe {dst.as_ptr().write_bytes(value, size);});diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rsindex 917a679b3..2e2f96c18 100644--- a/crates/backend-uzu/src/backends/cpu/context.rs+++ b/crates/backend-uzu/src/backends/cpu/context.rs@@ -37,8 +37,9 @@ impl Context for CpuContext {fn recommended_async_batch_size(&self,- _model_path: &Path,+ model_path: &Path,) -> usize {+ let _ = model_path;1}@@ -88,8 +89,9 @@ impl Context for CpuContext {fn start_capture(&self,- _trace_path: &std::path::Path,+ trace_path: &std::path::Path,) -> Result<(), CpuError> {+ let _ = trace_path;Err(CpuError::NotSupported)}diff --git a/crates/backend-uzu/src/backends/cpu/dense_buffer.rs b/crates/backend-uzu/src/backends/cpu/dense_buffer.rsindex e5b39b3ac..507c1ec88 100644--- a/crates/backend-uzu/src/backends/cpu/dense_buffer.rs+++ b/crates/backend-uzu/src/backends/cpu/dense_buffer.rs@@ -16,8 +16,9 @@ impl Buffer for UnsafeCell<Pin<Box<[u8]>>> {fn set_label(&mut self,- _label: Option<&str>,+ label: Option<&str>,) {+ let _ = label;}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 33aa51c67..fe73bacc4 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -4,7 +4,7 @@ use crate::{DataType,backends::{common::{- Encoder,+ AsBufferRangeMut, AsBufferRangeRef, Encoder,kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},},cpu::{Cpu, context::CpuContext},@@ -33,28 +33,52 @@ impl MatmulKernel for MatmulCpuKernel {fn encode(&mut self,- _context: &CpuContext,arguments: MatmulArguments<Cpu>,encoder: &mut Encoder<Cpu>,) {+ let MatmulArguments {+ a,+ a_offset,+ b,+ ab_scale,+ c,+ d,+ batch_dim,+ input_dim,+ output_dim,+ } = arguments;let command_buffer = encoder.as_command_buffer_mut();- let m = arguments.batch_dim as usize;- let n = arguments.output_dim as usize;- let k = arguments.input_dim as usize;+ let m = batch_dim as usize;+ let n = output_dim as usize;+ let k = input_dim as usize;let lda = k;let ldb = k;let ldd = n;- let a_offset = arguments.a_offset as usize;let data_type = self.data_type;+ let a_buffer_range = a.as_buffer_range_ref();+ let b_buffer_range = b.as_buffer_range_ref();+ let d_buffer_range = d.as_buffer_range_mut();+ let a_offset = a_buffer_range.range().start + a_offset;+ let b_offset = b_buffer_range.range().start;+ let d_offset = d_buffer_range.range().start;- let a_ptr = SendPtr(unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset));- let b_ptr = SendPtr(unsafe { &*arguments.b.get() }.as_ptr());- let ab_scale = arguments.ab_scale;- let d_ptr = SendPtrMut(arguments.d.get_mut().as_mut_ptr());+ let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));+ let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(b_offset));+ let d_ptr =+ SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_offset) as *mut u8 });- let (is_accumulate, bias_ptr) = match arguments.c {+ let (is_accumulate, bias_ptr) = match c {MatmulArgumentC::Accumulate => (true, None),- MatmulArgumentC::Bias(bias) => (false, Some(SendPtr(unsafe { &*bias.get() }.as_ptr()))),+ MatmulArgumentC::Bias(bias) => {+ let bias_buffer_range = bias.as_buffer_range_ref();+ let bias_range = bias_buffer_range.range();+ (+ false,+ Some(SendPtr(+ unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start),+ )),+ )+ },MatmulArgumentC::None => (false, None),};diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rsindex bbdde8210..b6ed115d4 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs@@ -1,5 +1,4 @@pub mod gemm;-pub mod gemv;mod matmul;pub use matmul::MatmulCpuKernel;diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rsindex d8ac4af5d..37258d949 100644--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs@@ -1,16 +1,16 @@use std::{cell::Cell, time::Duration};use metal::{- MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,- MTLCommandBufferHandler, MTLCommandBufferStatus, MTLCommandEncoder, MTLComputeCommandEncoder, MTLEvent,+ MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler,+ MTLCommandBufferStatus, MTLCommandEncoder, MTLComputeCommandEncoder, MTLEvent,};use objc2::{Message, rc::Retained, runtime::ProtocolObject};use super::Metal;use crate::backends::{common::{- AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,- CommandBufferInitial, CommandBufferPending,+ AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,},metal::error::MetalError,};@@ -120,28 +120,27 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {fn encode_copy(&mut self,- src: &Retained<ProtocolObject<dyn MTLBuffer>>,- src_range: std::ops::Range<usize>,- dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,- dst_range: std::ops::Range<usize>,+ src: BufferRangeRef<'_, <Metal as Backend>::DenseBuffer>,+ dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,) {+ let src_range = src.range();+ let dst_range = dst.range();let size = src_range.end - src_range.start;assert_eq!(size, dst_range.end - dst_range.start);- assert!(src.length() >= src_range.end && dst.length() >= dst_range.end);- self.ensure_blit().copy_buffer_to_buffer(src, src_range.start, dst, dst_range.start, size);+ self.ensure_blit().copy_buffer_to_buffer(src.buffer(), src_range.start, dst.buffer(), dst_range.start, size);}fn encode_fill(&mut self,- dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,- range: std::ops::Range<usize>,+ dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,value: u8,) {- assert!(range.end > range.start && range.end <= dst.length());+ let range = dst.range();+ assert!(range.end > range.start);assert!(range.start % 4 == 0 && range.end % 4 == 0);- self.ensure_blit().fill_buffer_range_value(dst, range, value);+ self.ensure_blit().fill_buffer_range_value(dst.buffer(), range, value);}fn encode_barrier(diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rsindex 1d31ba2cd..3350dfe46 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.rs@@ -1,10 +1,7 @@use crate::{DataType,backends::{- common::{- Context,- kernel::matmul::{MatmulArgumentC, MatmulArguments},- },+ common::{Context, kernel::matmul::MatmulArgumentC},metal::{Metal, context::MetalContext},},};@@ -127,16 +124,19 @@ impl GemmSpecialization {pub fn select(context: &MetalContext,data_type: DataType,- arguments: &MatmulArguments<Metal>,+ batch_dim: u32,+ input_dim: u32,+ output_dim: u32,+ c: &MatmulArgumentC<Metal>,) -> Self {- let overall_work_elements = arguments.batch_dim * arguments.output_dim;+ let overall_work_elements = batch_dim * output_dim;let is_float32 = matches!(data_type, DataType::F32);let prefer_half_or_tf32 = !is_float32 || MetalContext::tf32_enabled();let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column) =if context.is_high_performance() && overall_work_elements >= (1 << 20) {if prefer_half_or_tf32 {- if 2 * std::cmp::max(arguments.batch_dim, arguments.output_dim) > arguments.input_dim {+ if 2 * std::cmp::max(batch_dim, output_dim) > input_dim {(64, 64, 16, 2, 2)} else {(64, 32, 32, 2, 2)@@ -156,9 +156,9 @@ impl GemmSpecialization {block_depth,simdgroups_per_row,simdgroups_per_column,- align_mn: (arguments.batch_dim % block_rows) == 0 && (arguments.output_dim % block_cols) == 0,- align_k: (arguments.input_dim % block_depth) == 0,- is_accumulate: matches!(arguments.c, MatmulArgumentC::Accumulate),+ align_mn: (batch_dim % block_rows) == 0 && (output_dim % block_cols) == 0,+ align_k: (input_dim % block_depth) == 0,+ is_accumulate: matches!(c, MatmulArgumentC::Accumulate),}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex ea430a374..bdb779ca4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -7,7 +7,7 @@ use crate::{DataType,backends::{common::{- Encoder,+ Allocation, Encoder,gpu_types::GemmParams,kernel::{TensorAddBiasKernel,@@ -136,33 +136,40 @@ impl MatmulMetalKernel {fn encode_gemv(&mut self,- context: &MetalContext,encoder: &mut Encoder<Metal>,arguments: MatmulArguments<Metal>,) -> Result<(), MatmulError<Metal>> {- let (is_accumulate, output_bias) = match arguments.c {+ let MatmulArguments {+ a,+ a_offset,+ b,+ ab_scale,+ c,+ d,+ batch_dim,+ input_dim,+ output_dim,+ } = arguments;++ let (is_accumulate, output_bias) = match c {MatmulArgumentC::Accumulate => (true, None),MatmulArgumentC::Bias(bias) => (false, Some(bias)),MatmulArgumentC::None => (false, None),};- let specialization = gemv::GemvSpecialization::select(- arguments.input_dim,- arguments.output_dim,- is_accumulate,- output_bias.is_some(),- );+ let specialization =+ gemv::GemvSpecialization::select(input_dim, output_dim, is_accumulate, output_bias.is_some());- self.get_or_create_gemv(context, specialization)?.encode(- arguments.b,- (arguments.a, arguments.a_offset as usize),+ self.get_or_create_gemv(encoder.context(), specialization)?.encode(+ b,+ (a, a_offset),output_bias,- arguments.d,- arguments.input_dim,- arguments.output_dim,- arguments.input_dim,- arguments.ab_scale,- arguments.batch_dim,+ d,+ input_dim,+ output_dim,+ input_dim,+ ab_scale,+ batch_dim,specialization.output_rows_per_threadgroup(),encoder,);@@ -172,51 +179,56 @@ impl MatmulMetalKernel {fn encode_gemm(&mut self,- context: &MetalContext,encoder: &mut Encoder<Metal>,arguments: MatmulArguments<Metal>,) -> Result<(), MatmulError<Metal>> {- let specialization = gemm::GemmSpecialization::select(context, self.data_type, &arguments);-- let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);- let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);+ let MatmulArguments {+ a,+ a_offset,+ b,+ ab_scale,+ c,+ d,+ batch_dim,+ input_dim,+ output_dim,+ } = arguments;++ let specialization =+ gemm::GemmSpecialization::select(encoder.context(), self.data_type, batch_dim, input_dim, output_dim, &c);++ let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);+ let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);let params = GemmParams {- M: arguments.batch_dim,- N: arguments.output_dim,- K: arguments.input_dim,- leading_dimension_a: arguments.input_dim,- leading_dimension_b: arguments.input_dim,- leading_dimension_d: arguments.output_dim,+ M: batch_dim,+ N: output_dim,+ K: input_dim,+ leading_dimension_a: input_dim,+ leading_dimension_b: input_dim,+ leading_dimension_d: output_dim,threadgroups_per_row,threadgroups_per_column,swizzle_log: 0,- aligned_inner_iterations: arguments.input_dim / specialization.block_depth,+ aligned_inner_iterations: input_dim / specialization.block_depth,use_morton: false,};- let kernel = self.get_or_create_gemm(context, specialization)?;+ let kernel = self.get_or_create_gemm(encoder.context(), specialization)?;kernel.encode(- (arguments.a, arguments.a_offset as usize),- arguments.b,- &mut *arguments.d,+ (a, a_offset),+ b,+ &mut *d,std::slice::from_ref(¶ms),threadgroups_per_row,threadgroups_per_column,- arguments.ab_scale,+ ab_scale,encoder,);- if let MatmulArgumentC::Bias(bias) = arguments.c {- self.bias_add.encode(- None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,- bias,- arguments.d,- arguments.output_dim,- arguments.batch_dim * arguments.output_dim,- encoder,- );+ if let MatmulArgumentC::Bias(bias) = c {+ self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);}Ok(())@@ -224,21 +236,26 @@ impl MatmulMetalKernel {fn encode_gemm_mpp(&mut self,- context: &MetalContext,encoder: &mut Encoder<Metal>,arguments: MatmulArguments<Metal>,) -> Result<(), MatmulError<Metal>> {- let is_accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);- let specialization = gemm_mpp::GemmMppSpecialization::select(- arguments.batch_dim,- arguments.output_dim,- arguments.input_dim,- is_accumulate,- arguments.ab_scale != 1.0,- );-- let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);- let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);+ let MatmulArguments {+ a,+ a_offset,+ b,+ ab_scale,+ c,+ d,+ batch_dim,+ input_dim,+ output_dim,+ } = arguments;+ let is_accumulate = matches!(c, MatmulArgumentC::Accumulate);+ let specialization =+ gemm_mpp::GemmMppSpecialization::select(batch_dim, output_dim, input_dim, is_accumulate, ab_scale != 1.0);++ let threadgroups_per_row = output_dim.div_ceil(specialization.block_cols);+ let threadgroups_per_column = batch_dim.div_ceil(specialization.block_rows);let max_threadgroups_per_dim = threadgroups_per_row.max(threadgroups_per_column);let min_threadgroups_per_dim = threadgroups_per_row.min(threadgroups_per_column);@@ -250,12 +267,12 @@ impl MatmulMetalKernel {let use_morton = min_threadgroups_per_dim > 1 && morton_total <= 4_u32.saturating_mul(actual_total);let params = GemmParams {- M: arguments.batch_dim,- N: arguments.output_dim,- K: arguments.input_dim,- leading_dimension_a: arguments.input_dim,- leading_dimension_b: arguments.input_dim,- leading_dimension_d: arguments.output_dim,+ M: batch_dim,+ N: output_dim,+ K: input_dim,+ leading_dimension_a: input_dim,+ leading_dimension_b: input_dim,+ leading_dimension_d: output_dim,threadgroups_per_row,threadgroups_per_column,swizzle_log: 0,@@ -269,27 +286,20 @@ impl MatmulMetalKernel {(threadgroups_per_row, threadgroups_per_column)};- let kernel = self.get_or_create_gemm_mpp(context, specialization)?;+ let kernel = self.get_or_create_gemm_mpp(encoder.context(), specialization)?;kernel.encode(- (arguments.a, arguments.a_offset as usize),- arguments.b,- &mut *arguments.d,+ (a, a_offset),+ b,+ &mut *d,std::slice::from_ref(¶ms),group_count_x,group_count_y,- arguments.ab_scale,+ ab_scale,encoder,);- if let MatmulArgumentC::Bias(bias) = arguments.c {- self.bias_add.encode(- None::<&<Metal as crate::backends::common::Backend>::DenseBuffer>,- bias,- arguments.d,- arguments.output_dim,- arguments.batch_dim * arguments.output_dim,- encoder,- );+ if let MatmulArgumentC::Bias(bias) = c {+ self.bias_add.encode(None::<&Allocation<Metal>>, bias, d, output_dim, batch_dim * output_dim, encoder);}Ok(())@@ -308,18 +318,15 @@ pub enum MatmulDispatchPath {impl MatmulMetalKernel {pub fn encode_with_path(&mut self,- context: &MetalContext,arguments: MatmulArguments<Metal>,encoder: &mut Encoder<Metal>,path: MatmulDispatchPath,) {match path {- MatmulDispatchPath::Auto => self.encode(context, arguments, encoder),- MatmulDispatchPath::Gemv => self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV"),- MatmulDispatchPath::Gemm => self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM"),- MatmulDispatchPath::GemmMpp => {- self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP")- },+ MatmulDispatchPath::Auto => self.encode(arguments, encoder),+ MatmulDispatchPath::Gemv => self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV"),+ MatmulDispatchPath::Gemm => self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM"),+ MatmulDispatchPath::GemmMpp => self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP"),}}}@@ -362,16 +369,15 @@ impl MatmulKernel for MatmulMetalKernel {fn encode(&mut self,- context: &MetalContext,arguments: MatmulArguments<Metal>,encoder: &mut Encoder<Metal>,) {if arguments.batch_dim <= max_gemv_batch_threshold() {- self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");- } else if self.is_mpp_eligible(context) {- self.encode_gemm_mpp(context, encoder, arguments).expect("Failed to encode GEMM MPP kernel");+ self.encode_gemv(encoder, arguments).expect("Failed to encode GEMV kernel");+ } else if self.is_mpp_eligible(encoder.context()) {+ self.encode_gemm_mpp(encoder, arguments).expect("Failed to encode GEMM MPP kernel");} else {- self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");+ self.encode_gemm(encoder, arguments).expect("Failed to encode GEMM kernel");}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex 5f375544b..d4f1eb964 100644--- a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -217,4 +217,4 @@ PUBLIC KERNEL(ShortConvTrie)(dst_state[tap_count - 1] = static_cast<T>(x);}}-}\ No newline at end of file+}diff --git a/crates/backend-uzu/src/classifier/classifier.rs b/crates/backend-uzu/src/classifier/classifier.rsindex ddf9a6997..7683b416a 100644--- a/crates/backend-uzu/src/classifier/classifier.rs+++ b/crates/backend-uzu/src/classifier/classifier.rs@@ -1,16 +1,14 @@-#[cfg(feature = "tracing")]-use std::{cell::RefCell, rc::Rc};use std::{collections::HashMap, path::Path, time::Instant};#[cfg(feature = "tracing")]use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{- DataType,- backends::common::{Backend, Encoder},+ DataType, allocation_to_vec,+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},config::ModelMetadata,- encodable_block::EncodingParameters,- forward_pass::state::{ArrayId, ForwardPassState},+ encodable_block::{EncodingParameters, LayerArguments},+ forward_pass::token_inputs::TokenInputs,session::types::Error,};@@ -34,10 +32,6 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {) -> Result<ClassificationOutput, Error> {let run_start = Instant::now();- #[cfg(feature = "tracing")]- let (logits, _traces) = self.forward_pass_with_traces(&token_ids, &token_positions)?;-- #[cfg(not(feature = "tracing"))]let logits = self.forward_pass(&token_ids, &token_positions)?;let forward_duration = run_start.elapsed().as_secs_f64();@@ -86,8 +80,16 @@ impl<B: Backend> Classifier<B> {&mut self,token_ids: &[u64],token_positions: &[usize],- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {- self.forward_pass(token_ids, token_positions)+ ) -> Result<(Box<[f32]>, ActivationTrace<B>), Error> {+ let num_labels = self.context.model_config.model_config.num_labels;+ let mut traces = ActivationTrace::new_classifier(+ self.context.context.as_ref(),+ &self.context.model_shape,+ token_ids.len(),+ num_labels,+ );+ let logits = self.forward_pass_impl(token_ids, token_positions, Some(&mut traces))?;+ Ok((logits, traces))}#[cfg(feature = "tracing")]@@ -95,51 +97,110 @@ impl<B: Backend> Classifier<B> {&mut self,token_ids: &[u64],token_positions: &[usize],- ) -> Result<(Box<[f32]>, Rc<RefCell<ActivationTrace<B>>>), Error> {- let num_labels = self.context.model_config.model_config.num_labels;- let mut state = ForwardPassState::new_classifier(- self.context.context.clone(),- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.shared_buffers.clone(),- token_ids,- token_positions,- num_labels,- );+ ) -> Result<Box<[f32]>, Error> {+ self.forward_pass_impl(token_ids, token_positions, None)+ }+ fn forward_pass_impl(+ &mut self,+ token_ids: &[u64],+ token_positions: &[usize],+ #[cfg(feature = "tracing")] trace: Option<&mut ActivationTrace<B>>,+ ) -> Result<Box<[f32]>, Error> {+ let batch_dim = token_ids.len();+ let token_inputs = TokenInputs::new_classifier(self.context.context.as_ref(), token_ids, token_positions);let encoding_params = EncodingParameters::new();+ #[cfg(feature = "tracing")]+ let mut trace = trace;let mut encoder = Encoder::<B>::new(self.context.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;- self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ let mut main = self+ .context+ .embed+ .encode_lookup(token_inputs.token_ids(), batch_dim, &mut encoder)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ main = self+ .context+ .embedding_norm+ .encode(&main, 0, batch_dim, &mut encoder)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]- {- let traces = state.traces().clone();- state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().embedding_norm().clone());+ if let Some(trace) = trace.as_deref_mut() {+ let embedding_norm = trace.embedding_norm_mut().allocation_mut();+ encoder.encode_copy(&main, .., embedding_norm, ..);}- for layer in self.context.layers.iter() {- layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ let mut shortcut = encoder+ .allocate_scratch(main.as_buffer_range_ref().range().len())+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ for (layer_index, layer) in self.context.layers.iter().enumerate() {+ let rope_type = layer.rope_type();+ main = layer+ .encode(+ LayerArguments {+ batch_dim,+ token_positions: token_inputs.token_positions(),+ token_parents: token_inputs.token_parents(),+ token_subtrie_ranges: None,+ attention_sinks: self.context.shared_buffers.attention_sinks(layer_index),+ rope_cosines: self.context.shared_buffers.rope_cosines(rope_type),+ rope_sines: self.context.shared_buffers.rope_sines(rope_type),+ rope_max_sequence_length: self.context.model_shape.context_length(),+ rope_dim: self.context.model_shape.rope_dim(),+ sampling_start: 0,+ sampling_length: batch_dim,+ cache_layer: None,+ #[cfg(feature = "tracing")]+ trace: trace.as_deref_mut().map(|traces| &mut traces.layer_results[layer_index]),+ },+ &encoding_params,+ main,+ &mut shortcut,+ &mut encoder,+ )+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;}- self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ main = self+ .context+ .output_norm+ .encode(&main, 0, batch_dim, &mut encoder)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]- {- let traces = state.traces().clone();- state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().output_norm.clone());+ if let Some(trace) = trace.as_deref_mut() {+ encoder.encode_copy(&main, .., trace.output_norm.allocation_mut(), ..);+ }+ let pooling = self+ .context+ .pooling+ .encode(batch_dim, &main, &mut encoder)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ #[cfg(feature = "tracing")]+ if let Some(trace) = trace.as_deref_mut() {+ let output_pooling = trace.output_pooling_mut().allocation_mut();+ encoder.encode_copy(&pooling, .., output_pooling, ..);+ }+ let logits =+ self.context.prediction_head.encode(pooling, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ #[cfg(feature = "tracing")]+ if let Some(trace) = trace.as_deref_mut() {+ encoder.encode_copy(&logits, .., trace.logits.allocation_mut(), ..);}- self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ let completed = encoder+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;- let logits = self.copy_logits_from_state(&state)?;+ let logits_result = self.copy_logits_from_allocation(&logits);+ drop(logits);+ drop(main);+ drop(shortcut);+ drop(completed);- let traces = state.traces().clone();- Ok((logits, traces))+ logits_result}#[cfg(not(feature = "tracing"))]@@ -148,49 +209,24 @@ impl<B: Backend> Classifier<B> {token_ids: &[u64],token_positions: &[usize],) -> Result<Box<[f32]>, Error> {- let num_labels = self.context.model_config.model_config.num_labels;- let mut state = ForwardPassState::new_classifier(- self.context.context.clone(),- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.shared_buffers.clone(),- token_ids,- token_positions,- num_labels,- );-- let encoding_params = EncodingParameters::new();-- let mut encoder = Encoder::<B>::new(self.context.context.as_ref())- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;-- self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- for layer in self.context.layers.iter() {- layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- }- self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;-- let logits = self.copy_logits_from_state(&state)?;-- Ok(logits)+ self.forward_pass_impl(token_ids, token_positions)}- fn copy_logits_from_state(+ fn copy_logits_from_allocation(&self,- state: &ForwardPassState<B>,+ logits: &Allocation<B>,) -> Result<Box<[f32]>, Error> {- let num_labels = self.context.model_config.model_config.num_labels;- let logits = state.array(ArrayId::ClassifierPredictionHeadLogits).view(&[num_labels]);-- match logits.data_type() {- DataType::F32 => Ok(logits.as_slice::<f32>().into()),- DataType::F16 => Ok(logits.as_slice::<half::f16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),- DataType::BF16 => Ok(logits.as_slice::<half::bf16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),+ let logits_data_type: DataType =+ self.context.model_config.model_config.prediction_head_config.readout_config.activation_precision().into();++ match logits_data_type {+ DataType::F32 => Ok(allocation_to_vec::<B, f32>(logits).into()),+ DataType::F16 => {+ Ok(allocation_to_vec::<B, half::f16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())+ },+ DataType::BF16 => {+ Ok(allocation_to_vec::<B, half::bf16>(logits).into_iter().map(|x| x.to_f32()).collect::<Box<[_]>>())+ },_ => Err(Error::UnableToDecodeText),}}diff --git a/crates/backend-uzu/src/classifier/classifier_context.rs b/crates/backend-uzu/src/classifier/classifier_context.rsindex 2fe163e94..b526b8884 100644--- a/crates/backend-uzu/src/classifier/classifier_context.rs+++ b/crates/backend-uzu/src/classifier/classifier_context.rs@@ -1,17 +1,14 @@-use std::{cell::RefCell, fs::File, path::Path, rc::Rc};+use std::{fs::File, path::Path, rc::Rc};use crate::{DataType,backends::common::{Backend, Context},classifier::ClassifierError,config::{ClassifierModelConfig, ModelMetadata},- encodable_block::{- Activation, ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, Rope,- },+ encodable_block::{ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, Rope},forward_pass::{model_shape::ModelShape,- scratch_buffers::ScratchBuffers,- state::{ArrayId, RopeType, SharedBuffers},+ state::{RopeType, SharedBuffers},},parameters::ParameterLoader,session::types::Error,@@ -20,8 +17,7 @@ use crate::{pub struct ClassifierContext<B: Backend> {pub context: Rc<B::Context>,- pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- pub scratch_buffers: ScratchBuffers<B>,+ pub shared_buffers: Rc<SharedBuffers<B>>,pub model_config: ClassifierModelConfig,pub model_shape: ModelShape,@@ -56,20 +52,18 @@ impl<B: Backend> ClassifierContext<B> {let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let root_loader_view = loader.tree();- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);let transformer_tree = root_loader_view.subtree("transformer").map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("transformer".to_string())))?;- {- let mut shared_bufs = shared_buffers.borrow_mut();- if let Some(global_rope) = &mut shared_bufs.global_rope {- global_rope.update_data(&transformer_tree, "global_rope");- }- if let Some(local_rope) = &mut shared_bufs.local_rope {- local_rope.update_data(&transformer_tree, "local_rope");- }+ if let Some(global_rope) = &mut shared_buffers.global_rope {+ global_rope.update_data(&transformer_tree, "global_rope");+ }+ if let Some(local_rope) = &mut shared_buffers.local_rope {+ local_rope.update_data(&transformer_tree, "local_rope");}+ let shared_buffers = Rc::new(shared_buffers);let data_type = decoder_config.layer_config@@ -154,15 +148,10 @@ impl<B: Backend> ClassifierContext<B> {context.as_ref(),data_type,classifier_model_config.model_config.transformer_config.output_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&output_norm_tree,).map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("output norm: {:?}", e))))?;- let context_length = classifier_model_config.model_config.context_length;- let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, context_length);-let embedding_norm_tree = root_loader_view.subtree("embedding_norm").map_err(|_| Error::Classifier(ClassifierError::WeightSubtreeNotFound("embedding_norm".to_string())))?;@@ -170,8 +159,6 @@ impl<B: Backend> ClassifierContext<B> {context.as_ref(),data_type,classifier_model_config.model_config.embedding_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&embedding_norm_tree,).map_err(|e| Error::Classifier(ClassifierError::KernelCreationFailed(format!("embedding norm: {:?}", e))))?;@@ -190,24 +177,13 @@ impl<B: Backend> ClassifierContext<B> {})?;let prediction_head_dense = <dyn Linear<B>>::new::<1>(&prediction_head_config.dense_config,- prediction_head_config.use_dense_bias,model_dim,[model_dim],context.as_ref(),&prediction_head_dense_tree,- ArrayId::ClassifierPooling,- ArrayId::ClassifierPredictionHeadDense,- );-- let prediction_head_activation = Activation::<B>::new(- &context,- prediction_head_data_type,- prediction_head_config.activation.clone(),- ArrayId::ClassifierPredictionHeadDense,- ArrayId::ClassifierPredictionHeadDense,).map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))})?;let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {@@ -217,8 +193,6 @@ impl<B: Backend> ClassifierContext<B> {context.as_ref(),prediction_head_data_type,prediction_head_config.normalization_config.clone(),- ArrayId::ClassifierPredictionHeadDense,- ArrayId::ClassifierPredictionHeadNorm,&prediction_head_norm_tree,).map_err(|e| {@@ -230,13 +204,10 @@ impl<B: Backend> ClassifierContext<B> {})?;let prediction_head_final_linear = <dyn Linear<B>>::new::<1>(&prediction_head_config.readout_config,- true,model_dim,[num_labels],context.as_ref(),&prediction_head_readout_tree,- ArrayId::ClassifierPredictionHeadNorm,- ArrayId::ClassifierPredictionHeadLogits,).map_err(|e| {Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head readout: {:?}", e)))@@ -254,19 +225,21 @@ impl<B: Backend> ClassifierContext<B> {})?;let prediction_head = ClassifierPredictionHead::new(- prediction_head_dense.map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))- })?,- prediction_head_activation,+ context.as_ref(),+ prediction_head_dense,+ prediction_head_config.activation.clone(),+ prediction_head_data_type,prediction_head_norm,prediction_head_final_linear,- num_labels,- );+ model_dim,+ )+ .map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))+ })?;Ok(Self {context,shared_buffers,- scratch_buffers,model_config: classifier_model_config.clone(),model_shape,embed,diff --git a/crates/backend-uzu/src/config/mod.rs b/crates/backend-uzu/src/config/mod.rsindex f674948da..acc0aaa7a 100644--- a/crates/backend-uzu/src/config/mod.rs+++ b/crates/backend-uzu/src/config/mod.rs@@ -29,7 +29,6 @@ pub use classifier_model::*;pub use common::*;pub use decoder::*;pub use decoder_layer::*;-#[allow(unused_imports)]pub use delta_net::*;pub use embedding::*;pub use error::*;diff --git a/crates/backend-uzu/src/encodable_block/activation.rs b/crates/backend-uzu/src/encodable_block/activation.rsdeleted file mode 100644index 55598deec..000000000--- a/crates/backend-uzu/src/encodable_block/activation.rs+++ /dev/null@@ -1,64 +0,0 @@-//! Activation encodable.--use std::ops::DerefMut;--use crate::{- DataType,- backends::common::{- ActivationConfig, Backend, Encoder,- gpu_types::ActivationType,- kernel::{ActivationKernel, Kernels},- },- forward_pass::state::{ArrayId, ForwardPassState},-};--pub struct Activation<B: Backend> {- kernel: <B::Kernels as Kernels>::ActivationKernel,- activation: ActivationConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,-}--impl<B: Backend> Activation<B> {- pub fn new(- context: &B::Context,- data_type: DataType,- config: ActivationConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,- ) -> Result<Self, B::Error> {- let kernel =- <B::Kernels as Kernels>::ActivationKernel::new(context, data_type, input_array_id == output_array_id)?;- Ok(Self {- kernel,- activation: config,- input_array_id,- output_array_id,- })- }-- pub fn encode(- &self,- state: &mut ForwardPassState<B>,- encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let input_array = state.array(self.input_array_id);- let output_array = state.array(self.output_array_id);-- let n = input_array.shape().iter().product::<usize>();- if self.activation.act_type() == ActivationType::IDENTITY {- panic!("Identity activation is not supported for kernel");- }-- let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());- let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());- self.kernel.encode(- input_buffer_borrow.as_deref(),- output_array.buffer().borrow_mut().deref_mut(),- n as u32,- self.activation.act_type(),- encoder,- );- Ok(())- }-}diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rsindex 411a3bfd0..a402b1099 100644--- a/crates/backend-uzu/src/encodable_block/attention.rs+++ b/crates/backend-uzu/src/encodable_block/attention.rs@@ -1,16 +1,14 @@//! Attention kernel encodable.-use std::{- collections::HashMap,- ops::{Deref, DerefMut},-};+use std::collections::HashMap;use itertools::iproduct;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,gpu_types::ring::RingParams,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,@@ -18,11 +16,7 @@ use crate::{attention::{AttentionGemmArguments, AttentionGemmBlock},},},- encodable_block::EncodingParameters,- forward_pass::{- kv_cache_layer::KVCacheLayerState,- state::{ArrayId, ForwardPassState},- },+ forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},};fn env_gemm_attention_enabled() -> bool {@@ -49,18 +43,23 @@ pub struct Attention<B: Backend> {update_kv_cache_inplace_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,gate_kernel: Option<<B::Kernels as Kernels>::SigmoidGateKernel>,gemm_block: AttentionGemmBlock<B>,- layer_index: usize,+ data_type: DataType,attention_scale: Option<f32>,- has_sinks: bool,is_causal: bool,sliding_window_size: Option<usize>,}+pub struct AttentionArguments<'a, B: Backend> {+ pub projection_step: usize,+ pub token_subtrie_ranges: Option<&'a Allocation<B>>,+ pub attention_sinks: Option<&'a Allocation<B>>,+ pub kv_cache_layer: Option<&'a mut KVCacheLayer<B>>,+}+impl<B: Backend> Attention<B> {pub fn new(context: &B::Context,data_type: DataType,- layer_index: usize,attention_scale: Option<f32>,has_sinks: bool,is_causal: bool,@@ -125,9 +124,8 @@ impl<B: Backend> Attention<B> {update_kv_cache_inplace_kernel,gate_kernel,gemm_block,- layer_index,+ data_type,attention_scale,- has_sinks,is_causal,sliding_window_size,})@@ -164,55 +162,40 @@ impl<B: Backend> Attention<B> {pub fn encode(&self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters,+ mut args: AttentionArguments<B>,+ qkv: &Allocation<B>,+ queries: &Allocation<B>,+ mut rotated_keys: Allocation<B>,+ gate: Option<&Allocation<B>>,+ suffix_length: usize,+ num_heads: usize,+ num_groups: usize,+ head_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let qkv_array = state.array(ArrayId::QKV);- let queries_array = state.array(ArrayId::RotatedQueries);- let rotated_keys_array = state.array(ArrayId::RotatedKeys);- let attention_output_array = state.array(ArrayId::AttentionOutput);- let partials_array = state.array(ArrayId::AttentionPartials);- let sums_array = state.array(ArrayId::AttentionSums);- let maxs_array = state.array(ArrayId::AttentionMaxs);-- let suffix_length = qkv_array.shape()[0];- let num_heads = queries_array.shape()[0];- let head_dim = queries_array.shape()[2];- let num_groups = rotated_keys_array.shape()[0];- let max_sequence_length = if state.cache_layers().is_some() {- state.array(ArrayId::Keys(self.layer_index)).shape()[1]- } else {- // For classifiers without KV cache, max_sequence_length is just suffix_length- suffix_length- };-- let is_trie = state.token_subtrie_ranges.is_some();-- let (segment_prefix_length, ring_params) = if let Some(cache_layers) = state.cache_layers() {- let projection_step = parameters.projection_step.unwrap_or(0);- let cache = cache_layers.borrow();- let layer = cache.data[self.layer_index]- .as_transformer()- .expect("Attention kernel expects transformer layer state");- let ring_params = match layer.state {- KVCacheLayerState::Windowed {- ring_offset,- ring_length,- window_length,- } => {- let overflow = (ring_length + projection_step).saturating_sub(window_length);- Some(RingParams {- ring_offset: ((ring_offset + overflow) % window_length) as u32,- ring_length: (ring_length + projection_step).min(window_length) as u32,- })- },- _ => None,+ ) -> Result<Allocation<B>, B::Error> {+ let is_trie = args.token_subtrie_ranges.is_some();++ let (max_sequence_length, segment_prefix_length, ring_params) =+ if let Some(layer) = args.kv_cache_layer.as_deref() {+ let max_sequence_length = layer.shape[0];+ let ring_params = match layer.state {+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length,+ } => {+ let overflow = (ring_length + args.projection_step).saturating_sub(window_length);+ Some(RingParams {+ ring_offset: ((ring_offset + overflow) % window_length) as u32,+ ring_length: (ring_length + args.projection_step).min(window_length) as u32,+ })+ },+ _ => None,+ };+ (max_sequence_length, layer.projected_segment_prefix_length(args.projection_step), ring_params)+ } else {+ (suffix_length, 0, None)};- (layer.projected_segment_prefix_length(projection_step), ring_params)- } else {- (0, None)- };let is_kv_cache_ring = ring_params.is_some();@@ -231,99 +214,31 @@ impl<B: Backend> Attention<B> {let variant =self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, is_trie, is_kv_cache_ring);- let gate_array = self.gate_kernel.as_ref().map(|_| state.array(ArrayId::Gate));-- let sinks_array = self.has_sinks.then(|| state.array(ArrayId::AttentionSinks(self.layer_index)));-- let rotated_keys_buf_rc = rotated_keys_array.buffer();- let mut rotated_keys_buf_borrow = rotated_keys_buf_rc.borrow_mut();-- let qkv_buf_rc = qkv_array.buffer();- let qkv_buf_borrow = qkv_buf_rc.borrow();+ let sinks_allocation = args.attention_sinks;+ let trie_allocation = args.token_subtrie_ranges;// Get KV cache buffers only if KV cache exists (LLM mode)- let has_kv_cache = state.cache_layers().is_some();-- let key_cache_array = has_kv_cache.then(|| state.array(ArrayId::Keys(self.layer_index)));- let key_cache_buf_rc = key_cache_array.as_ref().map(|a| a.buffer());- let mut key_cache_buf_borrow = key_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());-- let value_cache_array = has_kv_cache.then(|| state.array(ArrayId::Values(self.layer_index)));- let value_cache_buf_rc = value_cache_array.as_ref().map(|a| a.buffer());- let mut value_cache_buf_borrow = value_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());-- let extracted_values_array = (!has_kv_cache).then(|| state.array(ArrayId::ExtractedValues));- let extracted_values_buf_rc = extracted_values_array.as_ref().map(|a| a.buffer());- let mut extracted_values_buf_borrow = extracted_values_buf_rc.as_ref().map(|rc| rc.borrow_mut());-- // For classifiers (no KV cache): extract values from QKV into a dedicated extracted_values buffer.- if !has_kv_cache {- self.update_kv_cache_inplace_kernel.encode(- None::<&B::DenseBuffer>,- qkv_buf_borrow.deref(),- // keys already in desired layout; harmless overwrite- rotated_keys_buf_borrow.deref_mut(),- extracted_values_buf_borrow.as_mut().unwrap().deref_mut(),- num_groups as u32,- num_heads as u32,- head_dim as u32,- suffix_length as u32,- 0u32,- max_sequence_length as u32,- encoder,- );- }-- let queries_buf_rc = queries_array.buffer();- let queries_buf_borrow = queries_buf_rc.borrow();-- let attention_output_buf_rc = attention_output_array.buffer();- let mut attention_output_buf_borrow = attention_output_buf_rc.borrow_mut();-- let trie_buf_rc = state.token_subtrie_ranges.as_ref().map(|a| a.buffer());- let trie_buf_borrow = trie_buf_rc.as_ref().map(|rc| rc.borrow());- let trie_buffer: Option<&B::DenseBuffer> = trie_buf_borrow.as_ref().map(|b| b.deref());-- let partials_buf_rc = partials_array.buffer();- let mut partials_buf_borrow = partials_buf_rc.borrow_mut();-- let sums_buf_rc = sums_array.buffer();- let mut sums_buf_borrow = sums_buf_rc.borrow_mut();-- let maxs_buf_rc = maxs_array.buffer();- let mut maxs_buf_borrow = maxs_buf_rc.borrow_mut();-- let sinks_buf_rc = sinks_array.as_ref().map(|b| b.buffer());- let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());- let sinks_buffer: Option<&B::DenseBuffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());-- // Only update KV cache for LLM mode (not for classifiers)- if has_kv_cache {- self.update_kv_cache_kernel.encode(- Some(rotated_keys_buf_borrow.deref()),- qkv_buf_borrow.deref(),- key_cache_buf_borrow.as_mut().unwrap().deref_mut(),- value_cache_buf_borrow.as_mut().unwrap().deref_mut(),- num_groups as u32,- num_heads as u32,- head_dim as u32,- suffix_length as u32,- segment_prefix_length as u32,- max_sequence_length as u32,- encoder,- );- }-- let key_cache_buffer: &B::DenseBuffer = if has_kv_cache {- key_cache_buf_borrow.as_ref().unwrap().deref()- } else {- rotated_keys_buf_borrow.deref()- };- let value_cache_buffer: &B::DenseBuffer = if has_kv_cache {- value_cache_buf_borrow.as_ref().unwrap().deref()- } else {- extracted_values_buf_borrow.as_ref().unwrap().deref()- };+ let has_kv_cache = args.kv_cache_layer.is_some();+ let extracted_values = (!has_kv_cache)+ .then(|| {+ let mut values =+ encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;+ self.update_kv_cache_inplace_kernel.encode(+ None::<&Allocation<B>>,+ qkv,+ &mut rotated_keys,+ &mut values,+ num_groups as u32,+ num_heads as u32,+ head_dim as u32,+ suffix_length as u32,+ 0u32,+ max_sequence_length as u32,+ encoder,+ );+ Ok(values)+ })+ .transpose()?;// KV cache layout: [max_sequence_length, num_groups, head_dim] (token-major)// For classifier mode (no KV cache) keys/values still come in group-major layout@@ -345,15 +260,96 @@ impl<B: Backend> Attention<B> {is_kv_cache_ring,};+ let (keys, values) = if let Some(layer) = args.kv_cache_layer.as_deref_mut() {+ self.update_kv_cache_kernel.encode(+ Some(&rotated_keys),+ qkv,+ &mut layer.keys,+ &mut layer.values,+ num_groups as u32,+ num_heads as u32,+ head_dim as u32,+ suffix_length as u32,+ segment_prefix_length as u32,+ max_sequence_length as u32,+ encoder,+ );+ (&layer.keys, &layer.values)+ } else {+ (&rotated_keys, extracted_values.as_ref().expect("Missing extracted values for classifier attention"))+ };++ let mut attention_output = self.encode_attention_variant(+ variant,+ &kernel_key,+ queries,+ keys,+ values,+ trie_allocation,+ sinks_allocation,+ gqa_factor,+ sequence_length,+ k_head_stride,+ k_seq_stride,+ v_head_stride,+ v_seq_stride,+ ring_params,+ scale,+ num_heads,+ num_groups,+ suffix_length,+ segment_prefix_length,+ max_sequence_length,+ head_dim,+ encoder,+ )?;++ if let Some(gate_kernel) = &self.gate_kernel {+ let total_elements = (suffix_length * num_heads * head_dim) as u32;+ let gate = gate.expect("Gate allocation not initialized");+ gate_kernel.encode(gate, &mut attention_output, total_elements, encoder);+ }++ Ok(attention_output)+ }++ #[allow(clippy::too_many_arguments)]+ fn encode_attention_variant(+ &self,+ variant: KernelVariant,+ kernel_key: &KernelKey,+ queries: &Allocation<B>,+ keys: &Allocation<B>,+ values: &Allocation<B>,+ trie_allocation: Option<&Allocation<B>>,+ sinks_allocation: Option<&Allocation<B>>,+ gqa_factor: usize,+ sequence_length: usize,+ k_head_stride: u32,+ k_seq_stride: u32,+ v_head_stride: u32,+ v_seq_stride: u32,+ ring_params: Option<RingParams>,+ scale: f32,+ num_heads: usize,+ num_groups: usize,+ suffix_length: usize,+ segment_prefix_length: usize,+ max_sequence_length: usize,+ head_dim: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let mut attention_output =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads * head_dim], self.data_type))?;match variant {KernelVariant::Gemm => {let args = AttentionGemmArguments {- queries_buffer: queries_buf_borrow.deref(),- keys_buffer: key_cache_buffer,- values_buffer: value_cache_buffer,- output_buffer: attention_output_buf_borrow.deref_mut(),- trie_buffer,- sinks_buffer,+ queries,+ keys,+ values,+ output: &mut attention_output,+ trie: trie_allocation,+ sinks: sinks_allocation,num_heads,num_groups,suffix_length,@@ -370,18 +366,18 @@ impl<B: Backend> Attention<B> {v_head_stride: v_head_stride as u64,v_seq_stride: v_seq_stride as u64,};- self.gemm_block.encode(state.context(), encoder, args).expect("Failed to encode AttentionGemmBlock");+ self.gemm_block.encode(encoder, args)?;},KernelVariant::SinglePass => {- let kernel = match self.single_pass_kernels.get(&kernel_key) {- Some(k) => k,- None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),- };+ let kernel = self+ .single_pass_kernels+ .get(kernel_key)+ .unwrap_or_else(|| panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key));kernel.encode(- queries_buf_borrow.deref(),- key_cache_buffer,- value_cache_buffer,- attention_output_buf_borrow.deref_mut(),+ queries,+ keys,+ values,+ &mut attention_output,gqa_factor as u32,sequence_length as u32,k_head_stride,@@ -390,30 +386,36 @@ impl<B: Backend> Attention<B> {v_seq_stride,ring_params,scale,- trie_buffer,+ trie_allocation,self.sliding_window_size.map(|s| s as u32),- sinks_buffer,+ sinks_allocation,num_heads as u32,suffix_length as u32,encoder,- )+ );},KernelVariant::TwoPass => {- let kernel_pass1 = match self.two_pass_1_kernels.get(&kernel_key) {- Some(k) => k,- None => panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key),- };- let kernel_pass2 = match self.two_pass_2_kernels.get(&(head_dim as u32)) {- Some(k) => k,- None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),- };+ let kernel_pass1 = self+ .two_pass_1_kernels+ .get(kernel_key)+ .unwrap_or_else(|| panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key));+ let kernel_pass2 = self+ .two_pass_2_kernels+ .get(&(head_dim as u32))+ .unwrap_or_else(|| panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key));+ let mut partials = encoder+ .allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32 * head_dim], DataType::F32))?;+ let mut sums =+ encoder.allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32], DataType::F32))?;+ let mut maxs =+ encoder.allocate_scratch(size_for_shape(&[num_heads * suffix_length * 32], DataType::F32))?;kernel_pass1.encode(- queries_buf_borrow.deref(),- key_cache_buffer,- value_cache_buffer,- partials_buf_borrow.deref_mut(),- sums_buf_borrow.deref_mut(),- maxs_buf_borrow.deref_mut(),+ queries,+ keys,+ values,+ &mut partials,+ &mut sums,+ &mut maxs,gqa_factor as u32,sequence_length as u32,k_head_stride,@@ -424,16 +426,16 @@ impl<B: Backend> Attention<B> {scale,num_heads as u32,suffix_length as u32,- trie_buffer,+ trie_allocation,self.sliding_window_size.map(|s| s as u32),- sinks_buffer,+ sinks_allocation,encoder,);kernel_pass2.encode(- partials_buf_borrow.deref(),- sums_buf_borrow.deref(),- maxs_buf_borrow.deref(),- attention_output_buf_borrow.deref_mut(),+ &partials,+ &sums,+ &maxs,+ &mut attention_output,num_heads as u32,suffix_length as u32,encoder,@@ -441,23 +443,11 @@ impl<B: Backend> Attention<B> {},}- if let Some(gate_kernel) = &self.gate_kernel {- let gate_array = gate_array.as_ref().unwrap();- let gate_buf_rc = gate_array.buffer();- let gate_buf_borrow = gate_buf_rc.borrow();- let total_elements = (suffix_length * num_heads * head_dim) as u32;- gate_kernel.encode(- gate_buf_borrow.deref(),- attention_output_buf_borrow.deref_mut(),- total_elements,- encoder,- );- }-- Ok(())+ Ok(attention_output)}}+#[derive(Clone, Copy)]enum KernelVariant {Gemm,SinglePass,diff --git a/crates/backend-uzu/src/encodable_block/classifier_layer.rs b/crates/backend-uzu/src/encodable_block/classifier_layer.rsindex 37e9b1401..e260cf96b 100644--- a/crates/backend-uzu/src/encodable_block/classifier_layer.rs+++ b/crates/backend-uzu/src/encodable_block/classifier_layer.rs@@ -2,19 +2,16 @@ use std::rc::Rc;use crate::{DataType,- backends::common::{Backend, Encoder},+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder, Kernels, kernel::TensorAddSwapKernel},config::TransformerLayerConfig,encodable_block::{- Attention, EncodingParameters, Linear, Mlp, Normalization, QKNorm, Rope, TensorAddSwap, TensorCopy,+ Attention, AttentionArguments, EncodingParameters, LayerArguments, Linear, Mlp, Normalization, QKNorm, Rope,},- forward_pass::state::{ArrayId, ForwardPassState},+ forward_pass::state::RopeType,parameters::ParameterTree,};pub struct ClassifierLayer<B: Backend> {- #[cfg_attr(not(feature = "tracing"), allow(dead_code))]- layer_index: usize,- copy_main_to_shortcut_mixer: TensorCopy<B>,pre_attention_norm: Option<Normalization<B>>,qkv_projection: Box<dyn Linear<B>>,qk_norm: Option<QKNorm<B>>,@@ -23,12 +20,15 @@ pub struct ClassifierLayer<B: Backend> {attention: Attention<B>,out_projection: Box<dyn Linear<B>>,post_attention_norm: Option<Normalization<B>>,- mixer_residual_add: TensorAddSwap<B>,- copy_main_to_shortcut_mlp: TensorCopy<B>,+ mixer_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,pre_mlp_norm: Normalization<B>,mlp: Box<dyn Mlp<B>>,post_mlp_norm: Option<Normalization<B>>,- mlp_residual_add: TensorAddSwap<B>,+ mlp_residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,+ model_dim: usize,+ num_heads: usize,+ num_groups: usize,+ head_dim: usize,}impl<B: Backend> ClassifierLayer<B> {@@ -47,12 +47,8 @@ impl<B: Backend> ClassifierLayer<B> {) -> Self {let ctx = context.as_ref(); // Reference for functions expecting &B::Contextlet attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");- let use_rope = attention_config.use_rope;let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();- let copy_main_to_shortcut_mixer =- TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();-let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {if layer_loader.subtree("pre_mixer_norm").is_ok() {Some(@@ -60,8 +56,6 @@ impl<B: Backend> ClassifierLayer<B> {ctx,intermediate_data_type,norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&layer_loader.subtree("pre_mixer_norm").unwrap(),).expect("Failed to create pre-attention norm kernel"),@@ -75,13 +69,10 @@ impl<B: Backend> ClassifierLayer<B> {let qkv_projection = <dyn Linear<B>>::new(&attention_config.qkv_projection_config,- attention_config.has_qkv_biases,model_dim,[num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],ctx,&layer_loader.subtree("mixer.qkv_projection").unwrap(),- ArrayId::Main,- ArrayId::QKV,).expect("Failed to create qkv projection");@@ -91,7 +82,6 @@ impl<B: Backend> ClassifierLayer<B> {intermediate_data_type,attention_config.query_norm_config.clone(),attention_config.key_norm_config.clone(),- ArrayId::QKV,&layer_loader.subtree("mixer").unwrap(),num_heads,num_groups,@@ -106,13 +96,10 @@ impl<B: Backend> ClassifierLayer<B> {let out_projection = <dyn Linear<B>>::new(&attention_config.out_projection_config,- attention_config.has_out_biases,num_heads * head_dim,[model_dim],ctx,&layer_loader.subtree("mixer.out_projection").unwrap(),- ArrayId::AttentionOutput,- ArrayId::Main,).expect("Failed to create out projection");@@ -122,8 +109,6 @@ impl<B: Backend> ClassifierLayer<B> {ctx,intermediate_data_type,norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&layer_loader.subtree("post_mixer_norm").unwrap(),).expect("Failed to create post-attention norm kernel"),@@ -132,18 +117,13 @@ impl<B: Backend> ClassifierLayer<B> {None};- let mixer_residual_add =- TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();-- let copy_main_to_shortcut_mlp =- TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();+ let mixer_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(ctx, intermediate_data_type)+ .expect("Failed to create mixer residual add kernel");let pre_mlp_norm = Normalization::new(ctx,intermediate_data_type,layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&layer_loader.subtree("pre_mlp_norm").unwrap(),).expect("Failed to create pre-MLP norm kernel");@@ -165,8 +145,6 @@ impl<B: Backend> ClassifierLayer<B> {ctx,intermediate_data_type,norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&layer_loader.subtree("post_mlp_norm").unwrap(),).expect("Failed to create post-MLP norm kernel"),@@ -178,7 +156,6 @@ impl<B: Backend> ClassifierLayer<B> {let attention = Attention::new(ctx,intermediate_data_type,- layer_index,attention_scale,attention_config.has_sinks,false,@@ -187,112 +164,163 @@ impl<B: Backend> ClassifierLayer<B> {).expect("Failed to create attention kernel");- let mlp_residual_add =- TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();+ let mlp_residual_add = <B::Kernels as Kernels>::TensorAddSwapKernel::new(ctx, intermediate_data_type)+ .expect("Failed to create mlp residual add kernel");Self {- layer_index,- copy_main_to_shortcut_mixer,pre_attention_norm,qkv_projection,qk_norm,rope,- use_rope,+ use_rope: attention_config.use_rope,attention,out_projection,post_attention_norm,mixer_residual_add,- copy_main_to_shortcut_mlp,pre_mlp_norm,mlp,post_mlp_norm,mlp_residual_add,+ model_dim,+ num_heads,+ num_groups,+ head_dim,}}pub fn encode(&self,- state: &mut ForwardPassState<B>,+ args: LayerArguments<B>,parameters: &EncodingParameters,+ main: Allocation<B>,+ shortcut: &mut Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {+ ) -> Result<Allocation<B>, B::Error> {+ let LayerArguments {+ batch_dim,+ token_positions,+ token_subtrie_ranges,+ attention_sinks,+ rope_cosines,+ rope_sines,+ rope_max_sequence_length,+ rope_dim,+ #[cfg(feature = "tracing")]+ trace,+ ..+ } = args;#[cfg(feature = "tracing")]- let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();+ let mut layer_traces = trace;+ let layer_len = batch_dim * self.model_dim;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.inputs.allocation_mut(), ..);}- self.copy_main_to_shortcut_mixer.encode(state, encoder)?;+ debug_assert_eq!(main.as_buffer_range_ref().range().len(), shortcut.as_buffer_range_ref().range().len());+ encoder.encode_copy(&main, .., shortcut, ..);- if let Some(ref pre_attn_norm) = self.pre_attention_norm {- pre_attn_norm.encode(state, encoder)?;- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());- }+ let mut main = if let Some(ref pre_attn_norm) = self.pre_attention_norm {+ pre_attn_norm.encode(&main, 0, batch_dim, encoder)?} else {- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());- }+ main+ };+ #[cfg(feature = "tracing")]+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.pre_attention_norm.allocation_mut(), ..);}- self.qkv_projection.encode(state, encoder)?;+ let mut qkv = self.qkv_projection.encode(main, batch_dim, encoder)?;if let Some(ref qk_norm) = self.qk_norm {- qk_norm.encode(state, encoder)?;+ qk_norm.encode(&mut qkv, batch_dim, encoder)?;}- self.rope.encode(state, self.use_rope, encoder)?;- self.attention.encode(state, parameters, encoder)?;- self.out_projection.encode(state, encoder)?;+ let cosines = rope_cosines.expect("Classifier attention layer requires RoPE cosine allocation");+ let sines = rope_sines.expect("Classifier attention layer requires RoPE sine allocation");+ let (queries, rotated_keys) = self.rope.encode(+ &qkv,+ token_positions,+ cosines,+ sines,+ batch_dim,+ self.num_heads,+ self.num_groups,+ self.head_dim,+ rope_max_sequence_length,+ rope_dim,+ self.use_rope,+ encoder,+ )?;+ let attention_output = self.attention.encode(+ AttentionArguments {+ projection_step: parameters.projection_step.unwrap_or(0),+ token_subtrie_ranges,+ attention_sinks,+ kv_cache_layer: None,+ },+ &qkv,+ &queries,+ rotated_keys,+ None,+ batch_dim,+ self.num_heads,+ self.num_groups,+ self.head_dim,+ encoder,+ )?;+ main = self.out_projection.encode(attention_output, batch_dim, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.attention.allocation_mut(), ..);}if let Some(ref post_attn_norm) = self.post_attention_norm {- post_attn_norm.encode(state, encoder)?;+ main = post_attn_norm.encode(&main, 0, batch_dim, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.post_attention_norm.allocation_mut(), ..);}}- self.mixer_residual_add.encode(state, encoder)?;+ self.mixer_residual_add.encode(&mut *shortcut, &mut main, layer_len as u32, encoder);#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.mlp_inputs.allocation_mut(), ..);}- self.copy_main_to_shortcut_mlp.encode(state, encoder)?;+ debug_assert_eq!(main.as_buffer_range_ref().range().len(), shortcut.as_buffer_range_ref().range().len());+ encoder.encode_copy(&main, .., shortcut, ..);- self.pre_mlp_norm.encode(state, encoder)?;+ main = self.pre_mlp_norm.encode(&main, 0, batch_dim, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.pre_mlp_norm.allocation_mut(), ..);}- self.mlp.encode(state, encoder)?;+ main = self.mlp.encode(main, batch_dim, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.mlp.allocation_mut(), ..);}if let Some(ref post_mlp_norm) = self.post_mlp_norm {- post_mlp_norm.encode(state, encoder)?;+ main = post_mlp_norm.encode(&main, 0, batch_dim, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.post_mlp_norm.allocation_mut(), ..);}}- self.mlp_residual_add.encode(state, encoder)?;+ self.mlp_residual_add.encode(shortcut, &mut main, layer_len as u32, encoder);#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&main, .., layer_traces.outputs.allocation_mut(), ..);}- Ok(())+ Ok(main)+ }++ pub fn rope_type(&self) -> RopeType {+ self.rope.rope_type()}}diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex a7565bc4c..e3d8f1948 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -4,14 +4,19 @@ use std::rc::Rc;use thiserror::Error;+#[cfg(feature = "tracing")]+use crate::forward_pass::traces::ActivationTrace;use crate::{DataType,- backends::common::{Backend, Encoder},+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},config::{DecoderConfig, DecoderLayerType, MixerConfig},- encodable_block::{Embedding, EncodingParameters, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError},+ encodable_block::{+ Embedding, EncodingParameters, LayerArguments, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError,+ },forward_pass::{+ cache_layers::CacheLayers,model_shape::ModelShape,- state::{ArrayId, ForwardPassState, RopeType},+ state::{RopeType, SharedBuffers},},parameters::ParameterTree,};@@ -31,6 +36,27 @@ pub struct Decoder<B: Backend> {pub norm: RMSNorm<B>,}+pub struct DecoderArguments<'a, B: Backend> {+ pub token_positions: &'a Allocation<B>,+ pub token_parents: &'a Allocation<B>,+ pub token_subtrie_ranges: Option<&'a Allocation<B>>,+ pub shared_buffers: &'a SharedBuffers<B>,+ pub cache_layers: Option<&'a mut CacheLayers<B>>,+ pub batch_dim: usize,+ pub sampling_start: usize,+ pub sampling_length: usize,+ pub rope_max_sequence_length: usize,+ pub rope_dim: usize,+ #[cfg(feature = "tracing")]+ pub trace: Option<&'a mut ActivationTrace<B>>,+}++pub enum DecoderDecodeInput<'a, B: Backend> {+ TokenIds(&'a Allocation<B>),+ #[cfg(metal_backend)]+ Embeddings(Allocation<B>),+}+impl<B: Backend> Decoder<B> {pub fn new(context: &B::Context,@@ -184,14 +210,11 @@ impl<B: Backend> Decoder<B> {context,norm_data_type,decoder_config.output_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&decoder_weight_loader.subtree("output_norm").unwrap(),None,- Some(ArrayId::Shortcut),+ true,true,)- .map(RMSNorm::with_sampling_range).expect("Failed to create output RMS norm kernel");(layers.into_boxed_slice(), norm_block)@@ -218,35 +241,140 @@ impl<B: Backend> Decoder<B> {})}- pub fn encode(+ fn run_layers(&self,- state: &mut ForwardPassState<B>,+ args: DecoderArguments<B>,+ mut main: Allocation<B>,parameters: &EncodingParameters,encoder: &mut Encoder<B>,- ) -> Result<(), DecoderError<B>> {- self.embed.encode_lookup(state, encoder)?;+ ) -> Result<(Allocation<B>, Allocation<B>), DecoderError<B>> {+ let DecoderArguments {+ token_positions,+ token_parents,+ token_subtrie_ranges,+ shared_buffers,+ mut cache_layers,+ batch_dim,+ sampling_start,+ sampling_length,+ rope_max_sequence_length,+ rope_dim,+ #[cfg(feature = "tracing")]+ trace,+ } = args;+ #[cfg(feature = "tracing")]+ let mut trace = trace;++ let mut shortcut =+ encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(DecoderError::BackendError)?;for layer in self.layers.iter() {- layer.encode(state, parameters, encoder).map_err(DecoderError::BackendError)?;- }+ let rope_type = layer.rope_type();+ let rope_cosines = rope_type.and_then(|rope_type| shared_buffers.rope_cosines(rope_type));+ let rope_sines = rope_type.and_then(|rope_type| shared_buffers.rope_sines(rope_type));+ let attention_sinks = shared_buffers.attention_sinks(layer.layer_index);+ #[cfg(feature = "tracing")]+ let layer_trace = trace.as_deref_mut().map(|trace| &mut trace.layer_results[layer.layer_index]);- if state.is_prefilling() {- return Ok(());+ let cache_layer = cache_layers.as_deref_mut().map(|cache_layers| &mut cache_layers.data[layer.layer_index]);+ main = layer+ .encode(+ LayerArguments {+ batch_dim,+ token_positions,+ token_parents,+ token_subtrie_ranges,+ attention_sinks,+ rope_cosines,+ rope_sines,+ rope_max_sequence_length,+ rope_dim,+ sampling_start,+ sampling_length,+ cache_layer,+ #[cfg(feature = "tracing")]+ trace: layer_trace,+ },+ parameters,+ main,+ &mut shortcut,+ encoder,+ )+ .map_err(DecoderError::BackendError)?;}- self.norm.encode(state, encoder).map_err(DecoderError::BackendError)?;+ Ok((main, shortcut))+ }++ pub fn encode_prefill(+ &self,+ args: DecoderArguments<B>,+ token_ids: &Allocation<B>,+ parameters: &EncodingParameters,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, DecoderError<B>> {+ let main = self.embed.encode_lookup(token_ids, args.batch_dim, encoder)?;+ let (main, _) = self.run_layers(args, main, parameters, encoder)?;+ Ok(main)+ }++ pub fn encode_decode<'input>(+ &self,+ args: DecoderArguments<B>,+ input: DecoderDecodeInput<'input, B>,+ hidden_capture: Option<&mut Allocation<B>>,+ parameters: &EncodingParameters,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, DecoderError<B>> {+ let sampling_start = args.sampling_start;+ let sampling_length = args.sampling_length;+ let batch_dim = args.batch_dim;+ #[cfg(feature = "tracing")]+ let mut trace = args.trace;+ let main = match input {+ DecoderDecodeInput::TokenIds(token_ids) => self.embed.encode_lookup(token_ids, args.batch_dim, encoder)?,+ #[cfg(metal_backend)]+ DecoderDecodeInput::Embeddings(main) => main,+ };+ let (main, mut shortcut) = self.run_layers(+ DecoderArguments {+ token_positions: args.token_positions,+ token_parents: args.token_parents,+ token_subtrie_ranges: args.token_subtrie_ranges,+ shared_buffers: args.shared_buffers,+ cache_layers: args.cache_layers,+ batch_dim: args.batch_dim,+ sampling_start,+ sampling_length,+ rope_max_sequence_length: args.rope_max_sequence_length,+ rope_dim: args.rope_dim,+ #[cfg(feature = "tracing")]+ trace: trace.as_deref_mut(),+ },+ main,+ parameters,+ encoder,+ )?;++ let output_norm = self+ .norm+ .encode(&main, sampling_start, sampling_length, Some(&mut shortcut), encoder)+ .map_err(DecoderError::BackendError)?;+ if let Some(hidden_capture) = hidden_capture {+ let row_size_bytes = shortcut.as_buffer_range_ref().range().len() / batch_dim;+ let input_offset = (batch_dim - 1) * row_size_bytes;+ encoder.encode_copy(&shortcut, input_offset..input_offset + row_size_bytes, hidden_capture, ..);+ }#[cfg(feature = "tracing")]- {- let traces = state.traces().clone();- state.encode_copy_array(encoder, ArrayId::Main, traces.borrow().output_norm.clone());+ if let Some(trace) = trace.as_deref_mut() {+ encoder.encode_copy(&output_norm, .., trace.output_norm.allocation_mut(), ..);}- self.embed.encode_readout(state, encoder)?;+ let logits = self.embed.encode_readout(sampling_length, &output_norm, encoder)?;#[cfg(feature = "tracing")]- {- let traces = state.traces().clone();- state.encode_copy_array(encoder, ArrayId::Logits, traces.borrow().logits.clone());+ if let Some(trace) = trace.as_deref_mut() {+ encoder.encode_copy(&logits, .., trace.logits.allocation_mut(), ..);}- Ok(())+ Ok(logits)}}diff --git a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rsindex 2e13af21d..6e107ef21 100644--- a/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/delta_net_mixer.rs@@ -1,11 +1,10 @@-use std::ops::{Deref, DerefMut};-use thiserror::Error;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{Conv1dPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel, DeltaNetNormGateKernel,DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,@@ -13,8 +12,8 @@ use crate::{},config::DeltaNetAttentionConfig,encodable_block::linear::{Linear, LinearBlockError},- forward_pass::state::{ArrayId, ForwardPassState},- parameters::{ParameterLoaderError, ParameterTree, resolve_subtree},+ forward_pass::delta_net_layer::DeltaNetLayer,+ parameters::{ParameterLoaderError, ParameterTree, try_resolve_subtree},};#[derive(Debug, Error)]@@ -30,7 +29,6 @@ pub enum DeltaNetMixerError<B: Backend> {}pub(crate) struct DeltaNetMixer<B: Backend> {- layer_index: usize,config: DeltaNetAttentionConfig,in_projection: Box<dyn Linear<B>>,out_projection: Box<dyn Linear<B>>,@@ -44,21 +42,26 @@ pub(crate) struct DeltaNetMixer<B: Backend> {delta_net_prefill: <B::Kernels as Kernels>::DeltaNetPrefillKernel,norm_gate: <B::Kernels as Kernels>::DeltaNetNormGateKernel,// Parameters- conv_weight: B::DenseBuffer,- conv_bias: Option<B::DenseBuffer>,- a_log: B::DenseBuffer,- dt_bias: B::DenseBuffer,- norm_weight: B::DenseBuffer,+ conv_weight: Allocation<B>,+ conv_bias: Option<Allocation<B>>,+ a_log: Allocation<B>,+ dt_bias: Allocation<B>,+ norm_weight: Allocation<B>,+ data_type: DataType,+}++pub(crate) struct DeltaNetArguments<'a, B: Backend> {+ pub active_row_count: usize,+ pub layer: &'a mut DeltaNetLayer<B>,}impl<B: Backend> DeltaNetMixer<B> {pub(crate) fn new(context: &B::Context,config: DeltaNetAttentionConfig,- layer_index: usize,model_dim: usize,decoder_layer_loader: &ParameterTree<B::Context>,- ) -> Result<Self, DeltaNetMixerError<B>> {+ ) -> Result<(Self, Option<Allocation<B>>), DeltaNetMixerError<B>> {if config.kernel_size < 2 {return Err(DeltaNetMixerError::UnsupportedConfiguration(format!("kernel_size must be >= 2, got {}",@@ -82,44 +85,38 @@ impl<B: Backend> DeltaNetMixer<B> {let has_bias = config.conv_config.has_biases;// Load weights- let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);- let conv_tree = resolve_subtree(&mixer_tree, &["conv", "conv1d"]);+ let mixer_tree = try_resolve_subtree(decoder_layer_loader, &["mixer"])?;+ let conv_tree = try_resolve_subtree(&mixer_tree, &["conv", "conv1d"])?;- let in_projection = <dyn Linear<B>>::new(+ let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&config.in_proj_config,- false,model_dim,[config.total_proj_dim()],context,- &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),- ArrayId::Main,- ArrayId::SsmInProj,+ &try_resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"])?,).map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;let out_projection = <dyn Linear<B>>::new(&config.out_proj_config,- false,config.value_dim(),[model_dim],context,- &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),- ArrayId::AttentionOutput,- ArrayId::Main,+ &try_resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"])?,).map_err(|e| DeltaNetMixerError::InnerLinearError(Box::new(e)))?;- let conv_weight = conv_tree.leaf("weights")?.read_buffer()?;+ let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;let conv_bias = if has_bias {- Some(conv_tree.leaf("biases")?.read_buffer()?)+ Some(conv_tree.leaf("biases")?.read_allocation()?)} else {None};- let a_log = mixer_tree.leaf("a_log")?.read_buffer()?;- let dt_bias = mixer_tree.leaf("dt_bias")?.read_buffer()?;- let norm_tree = resolve_subtree(&mixer_tree, &["norm", "inner_norm"]);- let norm_weight = norm_tree.leaf("scales")?.read_buffer()?;+ let a_log = mixer_tree.leaf("a_log")?.read_allocation()?;+ let dt_bias = mixer_tree.leaf("dt_bias")?.read_allocation()?;+ let norm_tree = try_resolve_subtree(&mixer_tree, &["norm", "inner_norm"])?;+ let norm_weight = norm_tree.leaf("scales")?.read_allocation()?;// Create kernelslet conv_update = <B::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(context, data_type, has_bias)@@ -140,41 +137,41 @@ impl<B: Backend> DeltaNetMixer<B> {let norm_gate = <B::Kernels as Kernels>::DeltaNetNormGateKernel::new(context, data_type).map_err(DeltaNetMixerError::BackendError)?;- Ok(Self {- layer_index,- config,- in_projection,- out_projection,- conv_update,- delta_net_update,- conv_pack,- conv_scan,- prefill_prep,- delta_net_prefill,- norm_gate,- conv_weight,- conv_bias,- a_log,- dt_bias,- norm_weight,- })+ Ok((+ Self {+ config,+ in_projection,+ out_projection,+ conv_update,+ delta_net_update,+ conv_pack,+ conv_scan,+ prefill_prep,+ delta_net_prefill,+ norm_gate,+ conv_weight,+ conv_bias,+ a_log,+ dt_bias,+ norm_weight,+ data_type,+ },+ in_projection_input_hadamard_factors,+ ))}fn run_conv_update(&self,- state: &mut ForwardPassState<B>,+ layer: &mut DeltaNetLayer<B>,+ in_proj: &mut Allocation<B>,encoder: &mut Encoder<B>,) {- let in_proj = state.array(ArrayId::SsmInProj);- let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));-let kernel_size = self.config.kernel_size;-self.conv_update.encode(&self.conv_weight,self.conv_bias.as_ref(),- in_proj.buffer().borrow_mut().deref_mut(),- conv_state.buffer().borrow_mut().deref_mut(),+ in_proj,+ &mut layer.conv_state,kernel_size as u32,self.config.conv_dim() as u32,(kernel_size - 1) as u32,@@ -184,24 +181,22 @@ impl<B: Backend> DeltaNetMixer<B> {fn run_conv_scan(&self,- state: &mut ForwardPassState<B>,+ layer: &mut DeltaNetLayer<B>,+ in_proj: &mut Allocation<B>,encoder: &mut Encoder<B>,suffix_length: usize,- ) {+ ) -> Result<(), B::Error> {let kernel_size = self.config.kernel_size;let state_stride = kernel_size - 1;let conv_dim = self.config.conv_dim();let total_proj_dim = self.config.total_proj_dim();-- let in_proj = state.array(ArrayId::SsmInProj);- let conv_state = state.array(ArrayId::DeltaNetConvState(self.layer_index));-- let padded = state.conv_padded_buffer().expect("Missing conv padded buffer");+ let mut padded = encoder+ .allocate_scratch(size_for_shape(&[suffix_length + state_stride, total_proj_dim], self.data_type))?;self.conv_pack.encode(- conv_state.buffer().borrow().deref(),- in_proj.buffer().borrow().deref(),- padded.buffer().borrow_mut().deref_mut(),+ &layer.conv_state,+ &*in_proj,+ &mut padded,state_stride as u32,total_proj_dim as u32,suffix_length as u32,@@ -210,11 +205,11 @@ impl<B: Backend> DeltaNetMixer<B> {);self.conv_scan.encode(- padded.buffer().borrow().deref(),+ &padded,&self.conv_weight,self.conv_bias.as_ref(),- in_proj.buffer().borrow_mut().deref_mut(),- conv_state.buffer().borrow_mut().deref_mut(),+ in_proj,+ &mut layer.conv_state,suffix_length as u32,kernel_size as u32,total_proj_dim as u32,@@ -223,24 +218,25 @@ impl<B: Backend> DeltaNetMixer<B> {total_proj_dim as u32,encoder,);+ Ok(())}fn run_delta_rule(&self,- state: &mut ForwardPassState<B>,+ layer: &mut DeltaNetLayer<B>,+ in_proj: &Allocation<B>,+ active_row_count: usize,encoder: &mut Encoder<B>,- ) {- let in_proj = state.array(ArrayId::SsmInProj);- let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));- let out = state.array(ArrayId::AttentionOutput);-+ ) -> Result<Allocation<B>, B::Error> {+ let mut out =+ encoder.allocate_scratch(size_for_shape(&[active_row_count, self.config.value_dim()], self.data_type))?;self.delta_net_update.encode(- in_proj.buffer().borrow().deref(),+ in_proj,&self.a_log,&self.dt_bias,&self.norm_weight,- ssm_state.buffer().borrow_mut().deref_mut(),- out.buffer().borrow_mut().deref_mut(),+ &mut layer.ssm_state,+ &mut out,self.config.num_heads as u32,self.config.num_groups as u32,self.config.value_head_dim as u32,@@ -249,34 +245,36 @@ impl<B: Backend> DeltaNetMixer<B> {self.config.norm_config.epsilon,encoder,);+ Ok(out)}fn run_delta_rule_prefill(&self,- state: &mut ForwardPassState<B>,- encoder: &mut Encoder<B>,+ layer: &mut DeltaNetLayer<B>,+ in_proj: &Allocation<B>,suffix_length: usize,- ) {+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {let num_dv_groups = ((self.config.value_head_dim + 7) / 8) as u32;-- let in_proj = state.array(ArrayId::SsmInProj);- let ssm_state = state.array(ArrayId::DeltaNetSsmState(self.layer_index));- let out = state.array(ArrayId::AttentionOutput);-- let aux = state.llm_aux.as_ref().expect("DeltaNet prep buffers not initialized");- let prep_q_norm = aux.delta_net_prep_q_norm.as_ref().expect("DeltaNet prep_q_norm not initialized");- let prep_k_norm = aux.delta_net_prep_k_norm.as_ref().expect("DeltaNet prep_k_norm not initialized");- let prep_beta = aux.delta_net_prep_beta.as_ref().expect("DeltaNet prep_beta not initialized");- let prep_decay = aux.delta_net_prep_decay.as_ref().expect("DeltaNet prep_decay not initialized");+ let mut prep_q_norm =+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;+ let mut prep_k_norm =+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.key_dim()], DataType::F32))?;+ let mut prep_beta =+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;+ let mut prep_decay =+ encoder.allocate_scratch(size_for_shape(&[suffix_length * self.config.num_heads], DataType::F32))?;+ let mut out =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.value_dim()], self.data_type))?;self.prefill_prep.encode(- in_proj.buffer().borrow().deref(),+ in_proj,&self.a_log,&self.dt_bias,- prep_q_norm.buffer().borrow_mut().deref_mut(),- prep_k_norm.buffer().borrow_mut().deref_mut(),- prep_beta.buffer().borrow_mut().deref_mut(),- prep_decay.buffer().borrow_mut().deref_mut(),+ &mut prep_q_norm,+ &mut prep_k_norm,+ &mut prep_beta,+ &mut prep_decay,self.config.num_heads as u32,self.config.num_groups as u32,self.config.key_dim() as u32,@@ -286,13 +284,13 @@ impl<B: Backend> DeltaNetMixer<B> {);self.delta_net_prefill.encode(- prep_q_norm.buffer().borrow().deref(),- prep_k_norm.buffer().borrow().deref(),- prep_beta.buffer().borrow().deref(),- prep_decay.buffer().borrow().deref(),- in_proj.buffer().borrow().deref(),- ssm_state.buffer().borrow_mut().deref_mut(),- out.buffer().borrow_mut().deref_mut(),+ &prep_q_norm,+ &prep_k_norm,+ &prep_beta,+ &prep_decay,+ in_proj,+ &mut layer.ssm_state,+ &mut out,self.config.num_heads as u32,self.config.num_groups as u32,self.config.value_head_dim as u32,@@ -303,10 +301,9 @@ impl<B: Backend> DeltaNetMixer<B> {encoder,);- // Norm gateself.norm_gate.encode(- out.buffer().borrow_mut().deref_mut(),- in_proj.buffer().borrow().deref(),+ &mut out,+ in_proj,&self.norm_weight,self.config.num_heads as u32,self.config.value_head_dim as u32,@@ -317,29 +314,31 @@ impl<B: Backend> DeltaNetMixer<B> {suffix_length as u32,encoder,);+ Ok(out)}pub(crate) fn encode(&self,- state: &mut ForwardPassState<B>,+ args: DeltaNetArguments<B>,+ input: Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let active_row_count = state.active_row_count();- if active_row_count == 0 {- return Ok(());- }-- self.in_projection.encode(state, encoder)?;-- if active_row_count == 1 {- self.run_conv_update(state, encoder);- self.run_delta_rule(state, encoder);+ ) -> Result<Allocation<B>, B::Error> {+ let DeltaNetArguments {+ active_row_count,+ layer,+ } = args;+ assert!(active_row_count > 0, "DeltaNet mixer requires at least one active row");++ let mut in_proj = self.in_projection.encode(input, active_row_count, encoder)?;++ let delta_output = if active_row_count == 1 {+ self.run_conv_update(layer, &mut in_proj, encoder);+ self.run_delta_rule(layer, &in_proj, active_row_count, encoder)?} else {- self.run_conv_scan(state, encoder, active_row_count);- self.run_delta_rule_prefill(state, encoder, active_row_count);- }+ self.run_conv_scan(layer, &mut in_proj, encoder, active_row_count)?;+ self.run_delta_rule_prefill(layer, &in_proj, active_row_count, encoder)?+ };- self.out_projection.encode(state, encoder)?;- Ok(())+ self.out_projection.encode(delta_output, active_row_count, encoder)}}diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 37d7de6f1..5d8e18bea 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -1,14 +1,11 @@-use std::{- cell::RefCell,- ops::{Deref, DerefMut},-};+use std::cell::RefCell;use thiserror::Error;use crate::{DataType,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{FullPrecisionEmbeddingLookupKernel, ManualKernels, QuantizedEmbeddingLookupKernel,matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},@@ -19,7 +16,6 @@ use crate::{},},config::EmbeddingConfig,- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLeaf, ParameterLoaderError, ParameterTree},};@@ -46,14 +42,14 @@ pub enum EmbeddingError<B: Backend> {enum TiedEmbeddingType<B: Backend> {FullPrecision {- weights: B::DenseBuffer,+ weights: Allocation<B>,lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,},Quantized {- weights: B::DenseBuffer,- scales: B::DenseBuffer,- biases: B::DenseBuffer,+ weights: Allocation<B>,+ scales: Allocation<B>,+ biases: Allocation<B>,lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,readout: QuantizedMatmulKernelEncodable<B>,},@@ -61,26 +57,26 @@ enum TiedEmbeddingType<B: Backend> {enum UntiedEmbeddingLookupType<B: Backend> {FullPrecision {- weights: B::DenseBuffer,+ weights: Allocation<B>,lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,},Quantized {- weights: B::DenseBuffer,- scales: B::DenseBuffer,- biases: B::DenseBuffer,+ weights: Allocation<B>,+ scales: Allocation<B>,+ biases: Allocation<B>,lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,},}enum UntiedEmbeddingReadoutType<B: Backend> {FullPrecision {- weights: B::DenseBuffer,+ weights: Allocation<B>,readout: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,},Quantized {- weights: B::DenseBuffer,- scales: B::DenseBuffer,- biases: B::DenseBuffer,+ weights: Allocation<B>,+ scales: Allocation<B>,+ biases: Allocation<B>,readout: QuantizedMatmulKernelEncodable<B>,},}@@ -98,6 +94,7 @@ enum EmbeddingTying<B: Backend> {pub struct Embedding<B: Backend> {tying: EmbeddingTying<B>,input_scale: f32,+ activation_data_type: DataType,vocab_size: u32,model_dim: u32,}@@ -151,6 +148,31 @@ fn validate_tensor<'file, 'context, 'leaf, B: Backend>(}impl<B: Backend> Embedding<B> {+ fn activation_data_type(config: &EmbeddingConfig) -> DataType {+ match config {+ EmbeddingConfig::Tied {+ precision,+ ..+ }+ | EmbeddingConfig::Untied {+ precision,+ ..+ } => (*precision).into(),+ EmbeddingConfig::MLXQuantizedTied {+ activation_precision,+ ..+ }+ | EmbeddingConfig::MLXQuantizedUntied {+ activation_precision,+ ..+ }+ | EmbeddingConfig::MLXSemiQuantizedUntied {+ activation_precision,+ ..+ } => (*activation_precision).into(),+ }+ }+pub fn new(context: &B::Context,vocab_size: u32,@@ -209,7 +231,7 @@ impl<B: Backend> Embedding<B> {let weights_leaf = lookup_tree.leaf("weights")?;validate_tensor(&weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;- let weights = weights_leaf.read_buffer()?;+ let weights = weights_leaf.read_allocation()?;let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;@@ -234,8 +256,8 @@ impl<B: Backend> Embedding<B> {let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;validate_tensor(&output_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;- let input_weights = input_weights_leaf.read_buffer()?;- let output_weights = output_weights_leaf.read_buffer()?;+ let input_weights = input_weights_leaf.read_allocation()?;+ let output_weights = output_weights_leaf.read_allocation()?;let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;@@ -277,9 +299,9 @@ impl<B: Backend> Embedding<B> {let biases_leaf = lookup_tree.leaf("biases")?;validate_tensor(&biases_leaf, [vocab_size as usize, num_groups], data_type)?;- let weights = weights_leaf.read_buffer()?;- let scales = scales_leaf.read_buffer()?;- let biases = biases_leaf.read_buffer()?;+ let weights = weights_leaf.read_allocation()?;+ let scales = scales_leaf.read_allocation()?;+ let biases = biases_leaf.read_allocation()?;let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(context,@@ -353,13 +375,13 @@ impl<B: Backend> Embedding<B> {let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;- let input_weights = input_weights_leaf.read_buffer()?;- let input_scales = input_scales_leaf.read_buffer()?;- let input_biases = input_biases_leaf.read_buffer()?;+ let input_weights = input_weights_leaf.read_allocation()?;+ let input_scales = input_scales_leaf.read_allocation()?;+ let input_biases = input_biases_leaf.read_allocation()?;- let output_weights = output_weights_leaf.read_buffer()?;- let output_scales = output_scales_leaf.read_buffer()?;- let output_biases = output_biases_leaf.read_buffer()?;+ let output_weights = output_weights_leaf.read_allocation()?;+ let output_scales = output_scales_leaf.read_allocation()?;+ let output_biases = output_biases_leaf.read_allocation()?;let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(context,@@ -430,11 +452,11 @@ impl<B: Backend> Embedding<B> {let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;- let input_weights = input_weights_leaf.read_buffer()?;+ let input_weights = input_weights_leaf.read_allocation()?;- let output_weights = output_weights_leaf.read_buffer()?;- let output_scales = output_scales_leaf.read_buffer()?;- let output_biases = output_biases_leaf.read_buffer()?;+ let output_weights = output_weights_leaf.read_allocation()?;+ let output_scales = output_scales_leaf.read_allocation()?;+ let output_biases = output_biases_leaf.read_allocation()?;let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type).map_err(EmbeddingError::BackendError)?;@@ -473,6 +495,7 @@ impl<B: Backend> Embedding<B> {};let input_scale = common.input_scale.unwrap_or(1.0);+ let activation_data_type = Self::activation_data_type(config);if let Some(logit_soft_cap) = common.logit_soft_cap {return Err(EmbeddingError::UnsupportedConfiguration(format!("logit_soft_cap={logit_soft_cap:?}")));@@ -481,6 +504,7 @@ impl<B: Backend> Embedding<B> {Ok(Self {tying,input_scale,+ activation_data_type,vocab_size,model_dim,})@@ -488,20 +512,17 @@ impl<B: Backend> Embedding<B> {pub fn encode_lookup(&self,- state: &mut ForwardPassState<B>,+ token_ids: &Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), EmbeddingError<B>> {- let batch_dim = state.active_row_count() as u32;-- let token_ids_array = state.array(ArrayId::TokenIds);- let token_ids_buffer_rc = token_ids_array.buffer();- let token_ids_buffer_borrow = token_ids_buffer_rc.borrow();- let token_ids = token_ids_buffer_borrow.deref();-- let output_array = state.array(ArrayId::Main);- let output_buffer_rc = output_array.buffer();- let mut output_buffer_borrow = output_buffer_rc.borrow_mut();- let output = output_buffer_borrow.deref_mut();+ ) -> Result<Allocation<B>, EmbeddingError<B>> {+ let mut output = encoder+ .allocate_scratch(crate::array::size_for_shape(+ &[batch_dim, self.model_dim as usize],+ self.activation_data_type,+ ))+ .map_err(EmbeddingError::BackendError)?;+ let batch_dim = batch_dim as u32;match &self.tying {EmbeddingTying::Tied {@@ -522,7 +543,7 @@ impl<B: Backend> Embedding<B> {} => lookup.encode(token_ids,weights,- output,+ &mut output,batch_dim,self.vocab_size,self.model_dim,@@ -554,7 +575,7 @@ impl<B: Backend> Embedding<B> {weights,scales,biases,- output,+ &mut output,batch_dim,self.vocab_size,self.model_dim,@@ -564,31 +585,22 @@ impl<B: Backend> Embedding<B> {},};- Ok(())+ Ok(output)}pub fn encode_readout(&self,- state: &mut ForwardPassState<B>,+ batch_dim: usize,+ input_allocation: &Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), EmbeddingError<B>> {- let batch_dim = state.sampling_length();-- if batch_dim == 0 {- return Ok(());- }-- let input_array = state.array(ArrayId::Main);- let input_buffer_rc = input_array.buffer();- let input_buffer_borrow = input_buffer_rc.borrow();- let input = input_buffer_borrow.deref();-- let output_array = state.array(ArrayId::Logits);- let output_buffer_rc = output_array.buffer();- let mut output_buffer_borrow = output_buffer_rc.borrow_mut();- let output = output_buffer_borrow.deref_mut();-- let input_offset = state.sampling_start() * self.model_dim as usize * input_array.data_type().size_in_bytes();+ ) -> Result<Allocation<B>, EmbeddingError<B>> {+ assert!(batch_dim > 0, "Embedding readout requires at least one row");+ let mut output_allocation = encoder+ .allocate_scratch(crate::array::size_for_shape(+ &[batch_dim, self.vocab_size as usize],+ self.activation_data_type,+ ))+ .map_err(EmbeddingError::BackendError)?;match &self.tying {EmbeddingTying::Tied {@@ -610,14 +622,13 @@ impl<B: Backend> Embedding<B> {let input_dim = self.model_dim as usize;let output_dim = self.vocab_size as usize;readout.borrow_mut().encode(- state.context(),MatmulArguments {- a: input,- a_offset: input_offset as u64,+ a: input_allocation,+ a_offset: 0,b: weights,ab_scale: 1.0,c: MatmulArgumentC::None,- d: output,+ d: &mut output_allocation,batch_dim: batch_dim as u32,input_dim: input_dim as u32,output_dim: output_dim as u32,@@ -648,19 +659,19 @@ impl<B: Backend> Embedding<B> {readout.encode(encoder,QuantizedMatmulArguments {- a_buffer: input,- a_offset: input_offset,- b_buffer: weights,- scales_buffer: scales,- zero_points_or_biases_buffer: biases,- output_buffer: output,+ a: input_allocation,+ a_offset: 0,+ b: weights,+ scales,+ zero_points_or_biases: biases,+ output: &mut output_allocation,hadamard_factors: None,batch_dim,},- )?;+ );},};- Ok(())+ Ok(output_allocation)}}diff --git a/crates/backend-uzu/src/encodable_block/layer/executables.rs b/crates/backend-uzu/src/encodable_block/layer/executables.rsindex b8dfe25a6..0bb930cb5 100644--- a/crates/backend-uzu/src/encodable_block/layer/executables.rs+++ b/crates/backend-uzu/src/encodable_block/layer/executables.rs@@ -1,26 +1,26 @@//! Layer executables - a single decoder layer with mixer, norms, and MLP.-#[cfg(feature = "tracing")]-use std::ops::{Deref, DerefMut};use std::rc::Rc;use super::MixerExecutables;#[cfg(feature = "tracing")]use crate::backends::common::{Kernels, kernel::TensorAddBiasKernel};+#[cfg(feature = "tracing")]+use crate::forward_pass::traces::LayerActivationTrace;use crate::{DataType,- backends::common::{Backend, Encoder},+ backends::common::{Allocation, AsBufferRangeRef, Backend, Encoder},config::{DecoderLayerConfig, DecoderLayerType, MixerConfig},encodable_block::{- Attention, DeltaNetMixer, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer,+ Attention, AttentionArguments, DeltaNetArguments, DeltaNetMixer, EncodingParameters, Linear, MambaArguments,+ MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvArguments, ShortConvMixer,},- forward_pass::state::{ArrayId, ForwardPassState},+ forward_pass::{cache_layers::CacheLayer, state::RopeType},parameters::ParameterTree,};/// A single decoder layer with all its components.pub struct LayerExecutables<B: Backend> {- #[cfg(feature = "tracing")]pub layer_index: usize,#[cfg(feature = "tracing")]pub tensor_add: <B::Kernels as Kernels>::TensorAddBiasKernel,@@ -30,6 +30,8 @@ pub struct LayerExecutables<B: Backend> {pub pre_mlp_norm: RMSNorm<B>,pub mlp: Box<dyn Mlp<B>>,pub post_mlp_norm: Option<RMSNorm<B>>,+ #[cfg(feature = "tracing")]+ model_dim: usize,}impl<B: Backend> LayerExecutables<B> {@@ -62,7 +64,6 @@ impl<B: Backend> LayerExecutables<B> {let (mixer, mixer_hadamard_factors) = match &layer_config.mixer_config {MixerConfig::Attention(attention_config) => {let rope_block = rope.expect("RoPE encoder missing for attention layer");- let use_rope = attention_config.use_rope;let layer_num_heads = attention_config.num_heads.unwrap_or(num_heads);let layer_num_groups = attention_config.num_groups.unwrap_or(num_groups);@@ -73,38 +74,58 @@ impl<B: Backend> LayerExecutables<B> {let (qkv_projection, input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&attention_config.qkv_projection_config,- attention_config.has_qkv_biases,model_dim,[q_dim, kv_dim, kv_dim],context,&decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),- ArrayId::Main,- ArrayId::QKV,).expect("Failed to create qkv projection");let has_gate = attention_config.has_gate || attention_config.gate_projection_config.is_some();- let gate_projection = if has_gate {+ let gate_projection = has_gate.then(|| {let gate_config = attention_config.gate_projection_config.as_ref().unwrap_or(&attention_config.qkv_projection_config);- Some(- <dyn Linear<B>>::new(- gate_config,+ let gate_tree = decoder_layer_loader.subtree("mixer.gate_projection").unwrap();+ match (input_hadamard_factors.is_some(), gate_config) {+ (+ true,+ crate::config::LinearConfig::RHTLinearWrapper {+ inner_config,+ ..+ },+ ) => {+ let output_factors = gate_tree+ .leaf("output_factors")+ .expect("Failed to get gate projection output_factors")+ .read_allocation()+ .expect("Failed to read gate projection output_factors");+ let inner_tree = gate_tree+ .subtree("inner_linear")+ .expect("Failed to get gate projection inner_linear subtree");+ <dyn Linear<B>>::new_with_output_hadamard(+ context,+ inner_config,+ &inner_tree,+ output_factors,+ model_dim,+ q_dim,+ )+ },+ (false,- model_dim,- [q_dim],- context,- &decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),- ArrayId::Main,- ArrayId::Gate,+ crate::config::LinearConfig::RHTLinearWrapper {+ ..+ },)- .expect("Failed to create gate projection"),- )- } else {- None- };+ | (true, _) => {+ panic!("attention qkv/gate projections must share input hadamard")+ },+ (false, _) => <dyn Linear<B>>::new(gate_config, model_dim, [q_dim], context, &gate_tree),+ }+ .expect("Failed to create gate projection")+ });let qk_norm =if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {@@ -113,7 +134,6 @@ impl<B: Backend> LayerExecutables<B> {intermediate_data_type,attention_config.query_norm_config.clone(),attention_config.key_norm_config.clone(),- ArrayId::QKV,&decoder_layer_loader.subtree("mixer").unwrap(),layer_num_heads,layer_num_groups,@@ -128,20 +148,16 @@ impl<B: Backend> LayerExecutables<B> {let out_projection = <dyn Linear<B>>::new(&attention_config.out_projection_config,- attention_config.has_out_biases,q_dim,[model_dim],context,&decoder_layer_loader.subtree("mixer.out_projection").unwrap(),- ArrayId::AttentionOutput,- ArrayId::Main,).expect("Failed to create out projection");let attention = Attention::new(context,intermediate_data_type,- layer_index,attention_scale,attention_config.has_sinks,attention_config.is_causal.unwrap_or(true),@@ -156,30 +172,31 @@ impl<B: Backend> LayerExecutables<B> {gate_projection,qk_norm,rope: rope_block,- use_rope,+ use_rope: attention_config.use_rope,attention,out_projection,+ num_heads: layer_num_heads,+ num_groups: layer_num_groups,+ head_dim: layer_head_dim,},input_hadamard_factors,)},MixerConfig::Mamba(mamba_config) => {- let mixer = MambaMixer::new(+ let (mixer, input_hadamard_factors) = MambaMixer::new(context,layer_type.clone(),mamba_config.clone(),layer_index,model_dim,- num_heads,- head_dim,- num_groups,decoder_layer_loader,- );+ )+ .expect("Failed to create Mamba mixer");(MixerExecutables::StateSpace {mixer,},- None,+ input_hadamard_factors,)},MixerConfig::ShortConv(short_conv_config) => {@@ -190,7 +207,8 @@ impl<B: Backend> LayerExecutables<B> {layer_index,model_dim,decoder_layer_loader,- );+ )+ .expect("Failed to create ShortConv mixer");(MixerExecutables::ShortConv {mixer,@@ -199,14 +217,14 @@ impl<B: Backend> LayerExecutables<B> {)},MixerConfig::DeltaNet(delta_net_config) => {- let mixer =- DeltaNetMixer::new(context, delta_net_config.clone(), layer_index, model_dim, decoder_layer_loader)+ let (mixer, input_hadamard_factors) =+ DeltaNetMixer::new(context, delta_net_config.clone(), model_dim, decoder_layer_loader).expect("Failed to create DeltaNet mixer");(MixerExecutables::DeltaNet {mixer,},- None,+ input_hadamard_factors,)},};@@ -215,11 +233,9 @@ impl<B: Backend> LayerExecutables<B> {context,intermediate_data_type,layer_config.pre_attention_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),mixer_hadamard_factors,- Some(ArrayId::Shortcut),+ true,layer_index > 0,).expect("Failed to create RMS norm kernel");@@ -230,11 +246,9 @@ impl<B: Backend> LayerExecutables<B> {context,intermediate_data_type,norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&decoder_layer_loader.subtree("post_mixer_norm").unwrap(),None,- None,+ false,false,).expect("Failed to create RMS norm kernel"),@@ -256,11 +270,9 @@ impl<B: Backend> LayerExecutables<B> {context,intermediate_data_type,layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),mlp_input_hadamard_factors,- Some(ArrayId::Shortcut),+ true,true,).expect("Failed to create RMS norm kernel");@@ -271,11 +283,9 @@ impl<B: Backend> LayerExecutables<B> {context,intermediate_data_type,norm_config.clone(),- ArrayId::Main,- ArrayId::Main,&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),None,- None,+ false,false,).expect("Failed to create RMS norm kernel"),@@ -285,7 +295,6 @@ impl<B: Backend> LayerExecutables<B> {};Self {- #[cfg(feature = "tracing")]layer_index,#[cfg(feature = "tracing")]tensor_add,@@ -295,26 +304,46 @@ impl<B: Backend> LayerExecutables<B> {pre_mlp_norm,mlp,post_mlp_norm,+ #[cfg(feature = "tracing")]+ model_dim,}}pub fn encode(&self,- state: &mut ForwardPassState<B>,+ args: LayerArguments<B>,parameters: &EncodingParameters,+ input: Allocation<B>,+ shortcut: &mut Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {+ ) -> Result<Allocation<B>, B::Error> {+ let LayerArguments {+ batch_dim,+ token_positions,+ token_parents,+ token_subtrie_ranges,+ attention_sinks,+ rope_cosines,+ rope_sines,+ rope_max_sequence_length,+ rope_dim,+ sampling_start,+ sampling_length,+ mut cache_layer,+ #[cfg(feature = "tracing")]+ trace,+ } = args;#[cfg(feature = "tracing")]- let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();+ let mut layer_traces = trace;- self.pre_attention_norm.encode(state, encoder)?;+ let mut hidden = self.pre_attention_norm.encode(&input, 0, batch_dim, Some(shortcut), encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().inputs.clone());- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(shortcut, .., layer_traces.inputs.allocation_mut(), ..);+ encoder.encode_copy(&hidden, .., layer_traces.pre_attention_norm.allocation_mut(), ..);}- match &self.mixer {+ hidden = match &self.mixer {MixerExecutables::Attention {qkv_projection,gate_projection,@@ -323,100 +352,201 @@ impl<B: Backend> LayerExecutables<B> {use_rope,attention,out_projection,+ num_heads,+ num_groups,+ head_dim,} => {- qkv_projection.encode(state, encoder)?;- if let Some(gate_proj) = gate_projection {- gate_proj.encode(state, encoder)?;- }+ let gate_input = if gate_projection.is_some() {+ let hidden_len = hidden.as_buffer_range_ref().range().len();+ let mut gate_input = encoder.allocate_scratch(hidden_len)?;+ encoder.encode_copy(&hidden, .., &mut gate_input, ..);+ Some(gate_input)+ } else {+ None+ };+ let mut qkv = qkv_projection.encode(hidden, batch_dim, encoder)?;+ let gate = match (gate_projection, gate_input) {+ (Some(gate_proj), Some(gate_input)) => Some(gate_proj.encode(gate_input, batch_dim, encoder)?),+ _ => None,+ };if let Some(norm) = qk_norm {- norm.encode(state, encoder)?;- }- rope.encode(state, *use_rope, encoder)?;- attention.encode(state, parameters, encoder)?;- out_projection.encode(state, encoder)?;- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());+ norm.encode(&mut qkv, batch_dim, encoder)?;}+ let cosines = rope_cosines.expect("Attention layer requires RoPE cosine allocation");+ let sines = rope_sines.expect("Attention layer requires RoPE sine allocation");+ let (queries, rotated_keys) = rope.encode(+ &qkv,+ token_positions,+ cosines,+ sines,+ batch_dim,+ *num_heads,+ *num_groups,+ *head_dim,+ rope_max_sequence_length,+ rope_dim,+ *use_rope,+ encoder,+ )?;+ let kv_cache_layer = cache_layer+ .as_deref_mut()+ .map(|layer| layer.as_transformer_mut().expect("Attention layer expects transformer cache"));+ let attention_output = attention.encode(+ AttentionArguments {+ projection_step: parameters.projection_step.unwrap_or(0),+ token_subtrie_ranges,+ attention_sinks,+ kv_cache_layer,+ },+ &qkv,+ &queries,+ rotated_keys,+ gate.as_ref(),+ batch_dim,+ *num_heads,+ *num_groups,+ *head_dim,+ encoder,+ )?;+ out_projection.encode(attention_output, batch_dim, encoder)?},MixerExecutables::StateSpace {mixer,} => {- mixer.encode(state, encoder)?;- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());- }+ let layer = cache_layer+ .as_deref_mut()+ .expect("State-space layer requires cache state")+ .as_state_space_mut()+ .expect("State-space mixer expects SSM cache layer");+ mixer.encode(+ MambaArguments {+ active_row_count: batch_dim,+ layer,+ },+ hidden,+ encoder,+ )?},MixerExecutables::ShortConv {mixer,} => {- mixer.encode(state, encoder)?;- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());- }+ let layer = cache_layer+ .as_deref_mut()+ .expect("ShortConv layer requires cache state")+ .as_short_conv_mut()+ .expect("ShortConv mixer expects ShortConv cache layer");+ mixer.encode(+ ShortConvArguments {+ active_row_count: batch_dim,+ sampling_start,+ sampling_length,+ token_parents,+ layer,+ },+ hidden,+ encoder,+ )?},MixerExecutables::DeltaNet {mixer,} => {- mixer.encode(state, encoder)?;- #[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());- }+ let layer = cache_layer+ .as_deref_mut()+ .expect("DeltaNet layer requires cache state")+ .as_delta_net_mut()+ .expect("DeltaNet mixer expects DeltaNet cache layer");+ mixer.encode(+ DeltaNetArguments {+ active_row_count: batch_dim,+ layer,+ },+ hidden,+ encoder,+ )?},+ };+ #[cfg(feature = "tracing")]+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&hidden, .., layer_traces.attention.allocation_mut(), ..);}if let Some(post_attention_norm) = &self.post_attention_norm {- post_attention_norm.encode(state, encoder)?;+ hidden = post_attention_norm.encode(&hidden, 0, batch_dim, None, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&hidden, .., layer_traces.post_attention_norm.allocation_mut(), ..);}}- // main = attention_result- self.pre_mlp_norm.encode(state, encoder)?;- // shortcut = attention_result + shortcut; main = normalized(shortcut)+ hidden = self.pre_mlp_norm.encode(&hidden, 0, batch_dim, Some(shortcut), encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Shortcut, layer_traces.borrow().mlp_inputs.clone());- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(shortcut, .., layer_traces.mlp_inputs.allocation_mut(), ..);+ encoder.encode_copy(&hidden, .., layer_traces.pre_mlp_norm.allocation_mut(), ..);}- self.mlp.encode(state, encoder)?;+ hidden = self.mlp.encode(hidden, batch_dim, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&hidden, .., layer_traces.mlp.allocation_mut(), ..);}if let Some(post_mlp_norm) = &self.post_mlp_norm {- post_mlp_norm.encode(state, encoder)?;+ hidden = post_mlp_norm.encode(&hidden, 0, batch_dim, None, encoder)?;#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ encoder.encode_copy(&hidden, .., layer_traces.post_mlp_norm.allocation_mut(), ..);}}- // main = mlp_result- // next layer's pre_attention_norm (or output_norm) fuses the residual add#[cfg(feature = "tracing")]- if let Some(ref layer_traces) = layer_traces {- let size = state.array(ArrayId::Main).shape().into_iter().copied().product::<usize>() as u32;- let input_a = state.array(ArrayId::Main).buffer();- let input_b = state.array(ArrayId::Shortcut).buffer();- let output = layer_traces.borrow().outputs.buffer();+ if let Some(layer_traces) = layer_traces.as_deref_mut() {+ let size = (batch_dim * self.model_dim) as u32;self.tensor_add.encode(- Some(input_a.borrow().deref()),- input_b.borrow().deref(),- output.borrow_mut().deref_mut(),+ Some(&hidden),+ &*shortcut,+ layer_traces.outputs.allocation_mut(),size,size,encoder,);}- Ok(())+ Ok(hidden)}++ pub fn rope_type(&self) -> Option<RopeType> {+ match &self.mixer {+ MixerExecutables::Attention {+ rope,+ ..+ } => Some(rope.rope_type()),+ MixerExecutables::StateSpace {+ ..+ }+ | MixerExecutables::ShortConv {+ ..+ }+ | MixerExecutables::DeltaNet {+ ..+ } => None,+ }+ }+}++pub struct LayerArguments<'a, B: Backend> {+ pub batch_dim: usize,+ pub token_positions: &'a Allocation<B>,+ pub token_parents: &'a Allocation<B>,+ pub token_subtrie_ranges: Option<&'a Allocation<B>>,+ pub attention_sinks: Option<&'a Allocation<B>>,+ pub rope_cosines: Option<&'a Allocation<B>>,+ pub rope_sines: Option<&'a Allocation<B>>,+ pub rope_max_sequence_length: usize,+ pub rope_dim: usize,+ pub sampling_start: usize,+ pub sampling_length: usize,+ pub cache_layer: Option<&'a mut CacheLayer<B>>,+ #[cfg(feature = "tracing")]+ pub trace: Option<&'a mut LayerActivationTrace<B>>,}diff --git a/crates/backend-uzu/src/encodable_block/layer/mixer.rs b/crates/backend-uzu/src/encodable_block/layer/mixer.rsindex e411bd700..c5b0db2d9 100644--- a/crates/backend-uzu/src/encodable_block/layer/mixer.rs+++ b/crates/backend-uzu/src/encodable_block/layer/mixer.rs@@ -17,6 +17,9 @@ pub(crate) enum MixerExecutables<B: Backend> {use_rope: bool,attention: Attention<B>,out_projection: Box<dyn Linear<B>>,+ num_heads: usize,+ num_groups: usize,+ head_dim: usize,},StateSpace {mixer: MambaMixer<B>,diff --git a/crates/backend-uzu/src/encodable_block/layer/mod.rs b/crates/backend-uzu/src/encodable_block/layer/mod.rsindex 5aea3c675..70b5019cb 100644--- a/crates/backend-uzu/src/encodable_block/layer/mod.rs+++ b/crates/backend-uzu/src/encodable_block/layer/mod.rs@@ -3,5 +3,5 @@mod executables;mod mixer;-pub use executables::LayerExecutables;+pub use executables::{LayerArguments, LayerExecutables};pub(crate) use mixer::MixerExecutables;diff --git a/crates/backend-uzu/src/encodable_block/layer_norm.rs b/crates/backend-uzu/src/encodable_block/layer_norm.rsindex 3469a7d1f..90a08cd16 100644--- a/crates/backend-uzu/src/encodable_block/layer_norm.rs+++ b/crates/backend-uzu/src/encodable_block/layer_norm.rs@@ -1,21 +1,15 @@//! LayerNorm encodable.-use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};-use thiserror::Error;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, LayerNormKernel},},config::{NormalizationConfig, UpcastMode},- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};@@ -30,9 +24,10 @@ pub enum LayerNormError<B: Backend> {pub struct LayerNorm<B: Backend> {kernel: <B::Kernels as Kernels>::LayerNormKernel,config: NormalizationConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,- scales_buffer: Rc<RefCell<B::DenseBuffer>>,+ scales: Allocation<B>,+ element_count: usize,+ input_data_type: DataType,+ output_data_type: DataType,}impl<B: Backend> LayerNorm<B> {@@ -40,64 +35,61 @@ impl<B: Backend> LayerNorm<B> {context: &B::Context,intermediate_data_type: DataType,config: NormalizationConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,) -> Result<Self, LayerNormError<B>> {- let scales = parameter_tree.leaf_array("scales").map_err(LayerNormError::ParameterError)?;+ let scales_leaf = parameter_tree.leaf("scales").map_err(LayerNormError::ParameterError)?;+ let element_count = scales_leaf.shape()[0];+ let scales = scales_leaf.read_allocation().map_err(LayerNormError::ParameterError)?;let accumulation_data_type: DataType = config.accumulation_precision.into();let scale_data_type: DataType = config.scale_precision.into();-let kernel = <B::Kernels as Kernels>::LayerNormKernel::new(context,intermediate_data_type,scale_data_type,scale_data_type,accumulation_data_type,- input_array_id == output_array_id,+ false,).map_err(LayerNormError::BackendError)?;Ok(Self {kernel,config,- input_array_id,- output_array_id,- scales_buffer: scales.buffer(),+ scales,+ element_count,+ input_data_type: intermediate_data_type,+ output_data_type: scale_data_type,})}pub fn encode(&self,- state: &mut ForwardPassState<B>,+ input: &Allocation<B>,+ row_offset: usize,+ row_count: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let input_array = state.array(self.input_array_id);- let output_array = state.array(self.output_array_id);-- let batch_dim = input_array.shape()[0] as u32;- let model_dim = input_array.shape()[1] as u32;+ ) -> Result<Allocation<B>, B::Error> {let full_layer = if self.config.upcast_mode == UpcastMode::FullLayer {1u32} else {0u32};-- let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());- let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());-+ let row_size = self.element_count * self.input_data_type.size_in_bytes();+ let input_offset = row_offset * row_size;+ let mut output =+ encoder.allocate_scratch(size_for_shape(&[row_count, self.element_count], self.output_data_type))?;self.kernel.encode(- input_buffer_borrow.as_deref(),- self.scales_buffer.borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),- batch_dim,- model_dim,+ Some((input, input_offset)),+ &self.scales,+ &mut output,+ row_count as u32,+ self.element_count as u32,self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),full_layer,encoder,);- Ok(())+ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs b/crates/backend-uzu/src/encodable_block/linear/full_precision.rsindex 065c43d44..32c2ec921 100644--- a/crates/backend-uzu/src/encodable_block/linear/full_precision.rs+++ b/crates/backend-uzu/src/encodable_block/linear/full_precision.rs@@ -1,22 +1,18 @@-use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};+use std::cell::RefCell;use thiserror::Error;use super::Linear;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{ManualKernels,matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},},},- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};@@ -53,12 +49,11 @@ pub enum FullPrecisionLinearError<B: Backend> {pub struct FullPrecisionLinear<B: Backend> {kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,- bias_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,- weights_buffer: Rc<RefCell<B::DenseBuffer>>,+ bias: Option<Allocation<B>>,+ weights: Allocation<B>,input_dim: usize,output_dim: usize,- input_array_id: ArrayId,- output_array_id: ArrayId,+ precision: DataType,}impl<B: Backend> FullPrecisionLinear<B> {@@ -68,15 +63,13 @@ impl<B: Backend> FullPrecisionLinear<B> {input_dim: usize,output_dim: usize,parameter_tree: &ParameterTree<B::Context>,- input_array_id: ArrayId,- output_array_id: ArrayId,) -> Result<Self, FullPrecisionLinearError<B>> {if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(FullPrecisionLinearError::UnsupportedDataType(precision));}- let weights = parameter_tree.leaf_array("weights").map_err(FullPrecisionLinearError::ParameterError)?;- let weights_shape = weights.shape().to_vec();+ let weights_leaf = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;+ let weights_shape = weights_leaf.shape().to_vec();if weights_shape != [output_dim, input_dim] {return Err(FullPrecisionLinearError::InvalidWeightsShape {got: weights_shape.into_boxed_slice(),@@ -85,16 +78,16 @@ impl<B: Backend> FullPrecisionLinear<B> {});}- if weights.data_type() != precision {+ if weights_leaf.data_type() != precision {return Err(FullPrecisionLinearError::InvalidWeightsDataType {expected: precision,- got: weights.data_type(),+ got: weights_leaf.data_type(),});}- let bias_buffer = match parameter_tree.leaf_array("biases") {- Ok(biases) => {- let bias_shape = biases.shape().to_vec();+ let bias = match parameter_tree.leaf("biases") {+ Ok(biases_leaf) => {+ let bias_shape = biases_leaf.shape().to_vec();if bias_shape != [output_dim] {return Err(FullPrecisionLinearError::InvalidBiasShape {got: bias_shape.into_boxed_slice(),@@ -102,14 +95,14 @@ impl<B: Backend> FullPrecisionLinear<B> {});}- if biases.data_type() != precision {+ if biases_leaf.data_type() != precision {return Err(FullPrecisionLinearError::InvalidBiasDataType {expected: precision,- got: biases.data_type(),+ got: biases_leaf.data_type(),});}- Some(biases.buffer())+ Some(biases_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?)},Err(_) => None,};@@ -118,12 +111,11 @@ impl<B: Backend> FullPrecisionLinear<B> {Ok(Self {kernel: RefCell::new(kernel),- bias_buffer,- weights_buffer: weights.buffer(),+ bias,+ weights: weights_leaf.read_allocation().map_err(FullPrecisionLinearError::ParameterError)?,input_dim,output_dim,- input_array_id,- output_array_id,+ precision,})}}@@ -131,32 +123,28 @@ impl<B: Backend> FullPrecisionLinear<B> {impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let batch_dim = state.active_row_count();- let input_array = state.array(self.input_array_id);- let output_array = state.array(self.output_array_id);-- let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());+ ) -> Result<Allocation<B>, B::Error> {+ let mut output = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.precision))?;self.kernel.borrow_mut().encode(- state.context(),MatmulArguments {- a: input_array.buffer().borrow().deref(),+ a: &input,a_offset: 0,- b: self.weights_buffer.borrow().deref(),+ b: &self.weights,ab_scale: 1.0,- c: match bias_borrow.as_deref() {- Some(b) => MatmulArgumentC::Bias(b),+ c: match self.bias.as_ref() {+ Some(bias) => MatmulArgumentC::Bias(bias),None => MatmulArgumentC::None,},- d: output_array.buffer().borrow_mut().deref_mut(),+ d: &mut output,batch_dim: batch_dim as u32,input_dim: self.input_dim as u32,output_dim: self.output_dim as u32,},encoder,);- Ok(())+ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex 8124cb042..33a88f2ba 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -10,18 +10,18 @@ pub use rht_wrapper::{RHTLinearWrapper, RHTLinearWrapperError};use thiserror::Error;use crate::{- backends::common::{Backend, Encoder},+ backends::common::{Allocation, Backend, Encoder},config::LinearConfig,- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};pub trait Linear<B: Backend> {fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error>;+ ) -> Result<Allocation<B>, B::Error>;}#[derive(Debug, Error)]@@ -30,10 +30,10 @@ pub enum LinearBlockError<B: Backend> {QuantizedLinearError(#[from] QuantizedLinearError<B>),#[error("FullPrecisionLinear error: {0}")]FullPrecisionLinearError(#[from] FullPrecisionLinearError<B>),- #[error("RHTLinearWrapper error: {0}")]- RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),#[error("QLoRALinearWrapper error: {0}")]QLoRALinearWrapperError(#[from] QLoRALinearWrapperError<B>),+ #[error("RHTLinearWrapper error: {0}")]+ RHTLinearWrapperError(#[from] RHTLinearWrapperError<B>),#[error("Parameter loading error: {0}")]ParameterError(#[from] ParameterLoaderError<B>),}@@ -41,13 +41,10 @@ pub enum LinearBlockError<B: Backend> {impl<B: Backend> dyn Linear<B> {pub fn new<const N: usize>(config: &LinearConfig,- _has_biases: bool,input_dimension: usize,output_dimensions: [usize; N],context: &B::Context,parameter_tree: &ParameterTree<B::Context>,- input_array_id: ArrayId,- output_array_id: ArrayId,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();match config {@@ -58,8 +55,6 @@ impl<B: Backend> dyn Linear<B> {input_dimension,output_dimension_sum,parameter_tree,- input_array_id,- output_array_id,None,)?;Ok(Box::new(block))@@ -73,8 +68,6 @@ impl<B: Backend> dyn Linear<B> {input_dimension,output_dimension_sum,parameter_tree,- input_array_id,- output_array_id,)?;Ok(Box::new(block))},@@ -92,37 +85,28 @@ impl<B: Backend> dyn Linear<B> {output_dimension_sum,parameter_tree,None,- input_array_id,- output_array_id,)?;Ok(Box::new(block))},LinearConfig::RHTLinearWrapper {block_size,inner_config,- } => {- let block = RHTLinearWrapper::new(- context,- *block_size,- inner_config,- input_dimension,- output_dimension_sum,- parameter_tree,- input_array_id,- output_array_id,- )?;- Ok(Box::new(block))- },+ } => Ok(Box::new(RHTLinearWrapper::new(+ context,+ *block_size,+ inner_config,+ input_dimension,+ output_dimension_sum,+ parameter_tree,+ )?)),}}pub fn new_with_output_hadamard(context: &B::Context,config: &LinearConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,- output_factors: B::DenseBuffer,+ output_factors: Allocation<B>,input_dim: usize,output_dim: usize,) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {@@ -133,8 +117,6 @@ impl<B: Backend> dyn Linear<B> {input_dim,output_dim,parameter_tree,- input_array_id,- output_array_id,Some(output_factors),)?)),LinearConfig::QLoRA {@@ -150,8 +132,6 @@ impl<B: Backend> dyn Linear<B> {output_dim,parameter_tree,Some(output_factors),- input_array_id,- output_array_id,)?)),inner_config => unimplemented!("{inner_config:?} doesn't support fused output hadamard"),}@@ -159,28 +139,23 @@ impl<B: Backend> dyn Linear<B> {pub fn new_extracting_input_hadamard<const N: usize>(config: &LinearConfig,- _has_biases: bool,input_dimension: usize,output_dimensions: [usize; N],context: &B::Context,parameter_tree: &ParameterTree<B::Context>,- input_array_id: ArrayId,- output_array_id: ArrayId,- ) -> Result<(Box<dyn Linear<B>>, Option<B::DenseBuffer>), LinearBlockError<B>> {+ ) -> Result<(Box<dyn Linear<B>>, Option<Allocation<B>>), LinearBlockError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();match config {LinearConfig::RHTLinearWrapper {inner_config,..} => {- let input_factors = parameter_tree.leaf("input_factors")?.read_buffer()?;- let output_factors = parameter_tree.leaf("output_factors")?.read_buffer()?;+ let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;+ let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;let inner_tree = parameter_tree.subtree("inner_linear")?;let inner_linear = Self::new_with_output_hadamard(context,inner_config,- input_array_id,- output_array_id,&inner_tree,output_factors,input_dimension,@@ -189,16 +164,7 @@ impl<B: Backend> dyn Linear<B> {Ok((inner_linear, Some(input_factors)))},other => {- let linear = Self::new(- other,- _has_biases,- input_dimension,- output_dimensions,- context,- parameter_tree,- input_array_id,- output_array_id,- )?;+ let linear = Self::new(other, input_dimension, output_dimensions, context, parameter_tree)?;Ok((linear, None))},}diff --git a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rsindex 93fc77d24..380899244 100644--- a/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/qlora_wrapper.rs@@ -1,14 +1,12 @@-use std::{- cell::RefCell,- ops::{Deref, DerefMut},-};+use std::cell::RefCell;use thiserror::Error;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{ManualKernels,matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},@@ -19,7 +17,6 @@ use crate::{Linear,linear::{LinearBlockError, QuantizedLinear, QuantizedLinearError},},- forward_pass::state::{ArrayId, ForwardPassState},prelude::{ParameterLeaf, ParameterLoaderError, ParameterTree},};@@ -45,14 +42,13 @@ pub enum QLoRALinearWrapperError<B: Backend> {pub struct QLoRALinearWrapper<B: Backend> {base_linear: QuantizedLinear<B>,adapter_kernel: RefCell<<B::Kernels as ManualKernels>::MatmulKernel>,- adapter_down: B::DenseBuffer,- adapter_up: B::DenseBuffer,+ adapter_down: Allocation<B>,+ adapter_up: Allocation<B>,input_dim: usize,output_dim: usize,lora_rank: usize,lora_scale: f32,- input_array_id: ArrayId,- output_array_id: ArrayId,+ data_type: DataType,}// TODO: figure out how to make this generic over QLoRAWrapperError::InvalidTensor or make one global "Invalid Tensor" error and make this a common helper@@ -85,9 +81,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {input_dim: usize,output_dim: usize,parameter_tree: &ParameterTree<B::Context>,- output_quantized_hadamard_factors: Option<B::DenseBuffer>,- input_array_id: ArrayId,- output_array_id: ArrayId,+ output_quantized_hadamard_factors: Option<Allocation<B>>,) -> Result<Self, QLoRALinearWrapperError<B>> {let data_type = quantization.activation_precision.into();@@ -97,8 +91,6 @@ impl<B: Backend> QLoRALinearWrapper<B> {input_dim,output_dim,parameter_tree,- input_array_id,- output_array_id,output_quantized_hadamard_factors,)?;let adapter_kernel =@@ -106,11 +98,11 @@ impl<B: Backend> QLoRALinearWrapper<B> {let adapter_down_leaf = parameter_tree.leaf("down_weights")?;validate_tensor(&adapter_down_leaf, [lora_rank as usize, input_dim as usize], data_type)?;- let adapter_down = adapter_down_leaf.read_buffer()?;+ let adapter_down = adapter_down_leaf.read_allocation()?;let adapter_up_leaf = parameter_tree.leaf("up_weights")?;validate_tensor(&adapter_up_leaf, [output_dim, lora_rank as usize], data_type)?;- let adapter_up = adapter_up_leaf.read_buffer()?;+ let adapter_up = adapter_up_leaf.read_allocation()?;Ok(Self {base_linear,@@ -121,8 +113,7 @@ impl<B: Backend> QLoRALinearWrapper<B> {output_dim,lora_rank,lora_scale,- input_array_id,- output_array_id,+ data_type,})}}@@ -130,27 +121,22 @@ impl<B: Backend> QLoRALinearWrapper<B> {impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), <B as Backend>::Error> {- self.base_linear.encode(state, encoder)?;-+ ) -> Result<Allocation<B>, <B as Backend>::Error> {let mut adapter_kernel = self.adapter_kernel.borrow_mut();- let batch_dim = state.active_row_count();-- let intermediate_array = state.common_aux.lora_intermediate.as_ref().unwrap();- let input_array = state.array(self.input_array_id);- let output_array = state.array(self.output_array_id);+ let mut intermediate =+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.lora_rank], self.data_type))?;adapter_kernel.encode(- state.context(),MatmulArguments {- a: input_array.buffer().borrow().deref(),+ a: &input,a_offset: 0,b: &self.adapter_down,ab_scale: 1.0,c: MatmulArgumentC::None,- d: intermediate_array.buffer().borrow_mut().deref_mut(),+ d: &mut intermediate,batch_dim: batch_dim as u32,input_dim: self.input_dim as u32,output_dim: self.lora_rank as u32,@@ -158,15 +144,16 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {encoder,);+ let mut output = self.base_linear.encode(input, batch_dim, encoder)?;+adapter_kernel.encode(- state.context(),MatmulArguments {- a: intermediate_array.buffer().borrow().deref(),+ a: &intermediate,a_offset: 0,b: &self.adapter_up,ab_scale: self.lora_scale,c: MatmulArgumentC::Accumulate,- d: output_array.buffer().borrow_mut().deref_mut(),+ d: &mut output,batch_dim: batch_dim as u32,input_dim: self.lora_rank as u32,output_dim: self.output_dim as u32,@@ -174,6 +161,6 @@ impl<B: Backend> Linear<B> for QLoRALinearWrapper<B> {encoder,);- Ok(())+ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/quantized.rs b/crates/backend-uzu/src/encodable_block/linear/quantized.rsindex 101cf36aa..4c1df9879 100644--- a/crates/backend-uzu/src/encodable_block/linear/quantized.rs+++ b/crates/backend-uzu/src/encodable_block/linear/quantized.rs@@ -1,16 +1,11 @@-use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};-use thiserror::Error;use super::Linear;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, TensorAddBiasKernel,quant_matmul::{@@ -20,7 +15,6 @@ use crate::{},},config::QuantizationConfig,- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};@@ -88,14 +82,13 @@ pub enum QuantizedLinearError<B: Backend> {pub struct QuantizedLinear<B: Backend> {kernel: QuantizedMatmulKernelEncodable<B>,bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,- biases_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,- weights_buffer: Rc<RefCell<B::DenseBuffer>>,- scales_buffer: Rc<RefCell<B::DenseBuffer>>,- zero_points_or_biases_buffer: Rc<RefCell<B::DenseBuffer>>,- output_hadamard_factors: Option<B::DenseBuffer>,+ biases: Option<Allocation<B>>,+ weights: Allocation<B>,+ scales: Allocation<B>,+ zero_points_or_biases: Allocation<B>,+ output_hadamard_factors: Option<Allocation<B>>,output_dim: usize,- input_array_id: ArrayId,- output_array_id: ArrayId,+ output_data_type: DataType,}impl<B: Backend> QuantizedLinear<B> {@@ -105,39 +98,37 @@ impl<B: Backend> QuantizedLinear<B> {input_dim: usize,output_dim: usize,parameter_tree: &ParameterTree<B::Context>,- input_array_id: ArrayId,- output_array_id: ArrayId,- output_hadamard_factors: Option<B::DenseBuffer>,+ output_hadamard_factors: Option<Allocation<B>>,) -> Result<Self, QuantizedLinearError<B>> {let kernel_data_type: DataType = config.activation_precision.into();if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));}- let weights = parameter_tree.leaf_array("weights").map_err(QuantizedLinearError::ParameterError)?;+ let weights_leaf = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;let packing_divisor = config.weight_quantization_mode.packing_divisor();let storage_type = config.weight_quantization_mode.storage_type();- if weights.data_type() != storage_type {+ if weights_leaf.data_type() != storage_type {return Err(QuantizedLinearError::InvalidWeightsDataType {expected: storage_type,- got: weights.data_type(),+ got: weights_leaf.data_type(),});}- let scales = parameter_tree.leaf_array("scales").map_err(QuantizedLinearError::ParameterError)?;- if scales.data_type() != kernel_data_type {+ let scales_leaf = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;+ if scales_leaf.data_type() != kernel_data_type {return Err(QuantizedLinearError::InvalidScalesDataType {expected: kernel_data_type,- got: scales.data_type(),+ got: scales_leaf.data_type(),});}let k_g = (input_dim + config.group_size - 1) / config.group_size;- let weights_shape = weights.shape().to_vec();- let scales_shape = scales.shape().to_vec();- let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf_array("deq_biases") {- Ok(deq_biases) => {- let deq_biases_shape = deq_biases.shape().to_vec();+ let weights_shape = weights_leaf.shape().to_vec();+ let scales_shape = scales_leaf.shape().to_vec();+ let (quantization_type, zero_points_or_biases) = match parameter_tree.leaf("deq_biases") {+ Ok(deq_biases_leaf) => {+ let deq_biases_shape = deq_biases_leaf.shape().to_vec();if !(weights_shape == [output_dim, input_dim / packing_divisor]&& scales_shape == [output_dim, k_g]&& deq_biases_shape == [output_dim, k_g])@@ -150,19 +141,22 @@ impl<B: Backend> QuantizedLinear<B> {});}- if deq_biases.data_type() != kernel_data_type {+ if deq_biases_leaf.data_type() != kernel_data_type {return Err(QuantizedLinearError::InvalidDeqBiasesDataType {expected: kernel_data_type,- got: deq_biases.data_type(),+ got: deq_biases_leaf.data_type(),});}- (QuantizedMatmulType::Mlx, deq_biases.buffer())+ (+ QuantizedMatmulType::Mlx,+ deq_biases_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,+ )},Err(_) => {- let zero_points =- parameter_tree.leaf_array("zero_points").map_err(QuantizedLinearError::ParameterError)?;- let zero_points_shape = zero_points.shape().to_vec();+ let zero_points_leaf =+ parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;+ let zero_points_shape = zero_points_leaf.shape().to_vec();let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;if !(weights_shape == [output_dim, input_dim / packing_divisor]&& scales_shape == [output_dim, k_g]@@ -177,20 +171,23 @@ impl<B: Backend> QuantizedLinear<B> {});}- if zero_points.data_type() != storage_type {+ if zero_points_leaf.data_type() != storage_type {return Err(QuantizedLinearError::InvalidZeroPointsDataType {expected: storage_type,- got: zero_points.data_type(),+ got: zero_points_leaf.data_type(),});}- (QuantizedMatmulType::ZeroPoint, zero_points.buffer())+ (+ QuantizedMatmulType::ZeroPoint,+ zero_points_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,+ )},};- let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf_array("biases") {- Ok(biases) => {- let bias_shape = biases.shape().to_vec();+ let (bias_add_kernel, biases) = match parameter_tree.leaf("biases") {+ Ok(biases_leaf) => {+ let bias_shape = biases_leaf.shape().to_vec();if bias_shape != [output_dim] {return Err(QuantizedLinearError::InvalidBiasShape {got: bias_shape.into_boxed_slice(),@@ -198,17 +195,20 @@ impl<B: Backend> QuantizedLinear<B> {});}- if biases.data_type() != kernel_data_type {+ if biases_leaf.data_type() != kernel_data_type {return Err(QuantizedLinearError::InvalidBiasDataType {expected: kernel_data_type,- got: biases.data_type(),+ got: biases_leaf.data_type(),});}let bias_add_kernel =<B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type, true).map_err(QuantizedLinearError::BackendError)?;- (Some(bias_add_kernel), Some(biases.buffer()))+ (+ Some(bias_add_kernel),+ Some(biases_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?),+ )},Err(_) => (None, None),};@@ -230,14 +230,13 @@ impl<B: Backend> QuantizedLinear<B> {Ok(Self {kernel,bias_add_kernel,- biases_buffer,- weights_buffer: weights.buffer(),- scales_buffer: scales.buffer(),- zero_points_or_biases_buffer,+ biases,+ weights: weights_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,+ scales: scales_leaf.read_allocation().map_err(QuantizedLinearError::ParameterError)?,+ zero_points_or_biases,output_hadamard_factors,output_dim,- input_array_id,- output_array_id,+ output_data_type: kernel_data_type,})}}@@ -245,43 +244,39 @@ impl<B: Backend> QuantizedLinear<B> {impl<B: Backend> Linear<B> for QuantizedLinear<B> {fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let batch_dim = state.active_row_count();-- let input_array = state.array(self.input_array_id);- let output_array = state.array(self.output_array_id);- let output_buf_rc = output_array.buffer();- let mut output_buf_borrow = output_buf_rc.borrow_mut();+ ) -> Result<Allocation<B>, B::Error> {+ let mut output =+ encoder.allocate_scratch(size_for_shape(&[batch_dim, self.output_dim], self.output_data_type))?;- self.kernel- .encode(- encoder,- QuantizedMatmulArguments {- a_buffer: input_array.buffer().borrow().deref(),- a_offset: 0,- b_buffer: self.weights_buffer.borrow().deref(),- scales_buffer: self.scales_buffer.borrow().deref(),- zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),- output_buffer: output_buf_borrow.deref_mut(),- hadamard_factors: self.output_hadamard_factors.as_ref(),- batch_dim,- },- )- .expect("Failed to encode quantized matmul");+ self.kernel.encode(+ encoder,+ QuantizedMatmulArguments {+ a: &input,+ a_offset: 0,+ b: &self.weights,+ scales: &self.scales,+ zero_points_or_biases: &self.zero_points_or_biases,+ output: &mut output,+ hadamard_factors: self.output_hadamard_factors.as_ref(),+ batch_dim,+ },+ );- if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {+ if let (Some(bias_add_kernel), Some(biases)) = (&self.bias_add_kernel, &self.biases) {let total_length = batch_dim * self.output_dim;bias_add_kernel.encode(- None::<&B::DenseBuffer>,- biases_buffer.borrow().deref(),- output_buf_borrow.deref_mut(),+ None::<&Allocation<B>>,+ biases,+ &mut output,self.output_dim as u32,total_length as u32,encoder,);}- Ok(())++ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rsindex f15033b35..2714551f4 100644--- a/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs+++ b/crates/backend-uzu/src/encodable_block/linear/rht_wrapper.rs@@ -1,15 +1,12 @@-use std::ops::DerefMut;-use thiserror::Error;use super::{Linear, LinearBlockError};use crate::{backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{HadamardTransformKernel, Kernels},},config::LinearConfig,- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};@@ -25,10 +22,9 @@ pub enum RHTLinearWrapperError<B: Backend> {pub struct RHTLinearWrapper<B: Backend> {input_hadamard_kernel: <B::Kernels as Kernels>::HadamardTransformKernel,- input_factors: B::DenseBuffer,+ input_factors: Allocation<B>,inner_linear: Box<dyn Linear<B>>,input_dimension: usize,- input_array_id: ArrayId,}impl<B: Backend> RHTLinearWrapper<B> {@@ -39,42 +35,30 @@ impl<B: Backend> RHTLinearWrapper<B> {input_dimension: usize,output_dimension: usize,parameter_tree: &ParameterTree<B::Context>,- input_array_id: ArrayId,- output_array_id: ArrayId,) -> Result<Self, RHTLinearWrapperError<B>> {assert!(block_size == 32, "only block size 32 hadamard is supported");let data_type = inner_config.activation_precision().into();-- let input_factors_leaf = parameter_tree.leaf("input_factors")?;- let output_factors_leaf = parameter_tree.leaf("output_factors")?;-- let input_factors = input_factors_leaf.read_buffer()?;- let output_factors = output_factors_leaf.read_buffer()?;-+ let input_factors = parameter_tree.leaf("input_factors")?.read_allocation()?;+ let output_factors = parameter_tree.leaf("output_factors")?.read_allocation()?;let input_hadamard_kernel = <B::Kernels as Kernels>::HadamardTransformKernel::new(context, data_type).map_err(RHTLinearWrapperError::BackendError)?;-let inner_linear_tree = parameter_tree.subtree("inner_linear")?;-let inner_linear = <dyn Linear<B>>::new_with_output_hadamard(context,inner_config,- input_array_id,- output_array_id,&inner_linear_tree,output_factors,input_dimension,output_dimension,)- .map_err(|e| RHTLinearWrapperError::InnerLinearError(Box::new(e)))?;+ .map_err(|error| RHTLinearWrapperError::InnerLinearError(Box::new(error)))?;Ok(Self {input_hadamard_kernel,input_factors,inner_linear,input_dimension,- input_array_id,})}}@@ -82,20 +66,17 @@ impl<B: Backend> RHTLinearWrapper<B> {impl<B: Backend> Linear<B> for RHTLinearWrapper<B> {fn encode(&self,- state: &mut ForwardPassState<B>,+ mut input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- // Input Hadamard- let input_array = state.array(self.input_array_id);+ ) -> Result<Allocation<B>, B::Error> {self.input_hadamard_kernel.encode(- input_array.buffer().borrow_mut().deref_mut(),+ &mut input,&self.input_factors,self.input_dimension as u32,- state.active_row_count() as u32,+ batch_dim as u32,encoder,);-- // Matmul with fused output Hadamard- self.inner_linear.encode(state, encoder)+ self.inner_linear.encode(input, batch_dim, encoder)}}diff --git a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs b/crates/backend-uzu/src/encodable_block/mamba_mixer.rsindex 361074d9c..b2b7bb66a 100644--- a/crates/backend-uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/mamba_mixer.rs@@ -1,28 +1,38 @@//! Mamba2 SSM mixer encodable.-use std::{- env,- ops::{Deref, DerefMut},-};+use std::env;++use thiserror::Error;use crate::{DataType,- array::Array,+ array::size_for_shape,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},config::{DecoderLayerType, Mamba2Config},- encodable_block::linear::Linear,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::{ParameterTree, resolve_subtree},+ encodable_block::linear::{Linear, LinearBlockError},+ forward_pass::ssm_layer::SSMLayer,+ parameters::{ParameterLoaderError, ParameterTree, try_resolve_subtree},};+#[derive(Debug, Error)]+pub enum MambaMixerError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Unsupported configuration: {0}")]+ UnsupportedConfiguration(String),+ #[error("Linear error: {0}")]+ LinearError(#[from] Box<LinearBlockError<B>>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+}+pub(crate) struct MambaMixer<B: Backend> {- layer_index: usize,config: Mamba2Config,in_projection: Box<dyn Linear<B>>,out_projection: Box<dyn Linear<B>>,@@ -32,11 +42,29 @@ pub(crate) struct MambaMixer<B: Backend> {conv_scan: <B::Kernels as Kernels>::Conv1dScanKernel,ssd_prefill: SSDPrefillKernels<B>,ssd_update: <B::Kernels as Kernels>::SSDUpdateKernel,- conv_weight: Array<B>,- conv_bias: Option<Array<B>>,- gate_bias: Array<B>,- skip_connection_weight: Array<B>,+ conv_weight: Allocation<B>,+ conv_bias: Option<Allocation<B>>,+ gate_bias: Allocation<B>,+ skip_connection_weight: Allocation<B>,prefill_mode: SSDPrefillMode,+ data_type: DataType,+}++pub(crate) struct MambaArguments<'a, B: Backend> {+ pub active_row_count: usize,+ pub layer: &'a mut SSMLayer<B>,+}++struct SplitInProjectionOutput<B: Backend> {+ conv_inputs: Allocation<B>,+ gate: Allocation<B>,+ time_step: Allocation<B>,+}++struct ConvScanOutput<B: Backend> {+ conv_x: Allocation<B>,+ state_b: Allocation<B>,+ state_c: Allocation<B>,}fn resolve_prefill_mode_from_env() -> SSDPrefillMode {@@ -58,155 +86,151 @@ impl<B: Backend> MambaMixer<B> {mamba_config: Mamba2Config,layer_index: usize,model_dim: usize,- num_heads: usize,- head_dim: usize,- num_groups: usize,decoder_layer_loader: &ParameterTree<B::Context>,- ) -> Self {- let _ = (num_heads, head_dim, num_groups);+ ) -> Result<(Self, Option<Allocation<B>>), MambaMixerError<B>> {if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {- panic!("Layer {} marked as transformer but Mamba mixer config provided", layer_index);+ return Err(MambaMixerError::UnsupportedConfiguration(format!(+ "layer {layer_index} marked as transformer but Mamba mixer config provided"+ )));}- let split_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);- let conv_tree = resolve_subtree(&split_tree, &["conv", "conv1d"]);+ let split_tree = try_resolve_subtree(decoder_layer_loader, &["mixer"])?;+ let conv_tree = try_resolve_subtree(&split_tree, &["conv", "conv1d"])?;let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();- let in_projection = <dyn Linear<B>>::new(+ let (in_projection, in_projection_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&mamba_config.in_projection_config,- mamba_config.has_in_biases,model_dim,[mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],context,- &resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),- ArrayId::Main,- ArrayId::SsmInProj,+ &try_resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"])?,)- .expect("Failed to create in-projection kernel");+ .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;let out_projection = <dyn Linear<B>>::new(&mamba_config.out_projection_config,- mamba_config.has_out_biases,mamba_config.inner_dim(),[model_dim],context,- &resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),- ArrayId::AttentionOutput,- ArrayId::Main,+ &try_resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"])?,)- .expect("Failed to create out-projection kernel");+ .map_err(|err| MambaMixerError::LinearError(Box::new(err)))?;- let conv_weight = conv_tree.leaf_array("weights").unwrap();+ let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;let conv_bias = if mamba_config.conv_config.has_biases {- Some(conv_tree.leaf_array("biases").unwrap())+ Some(conv_tree.leaf("biases")?.read_allocation()?)} else {None};- let gate_bias = split_tree.leaf_array("gate_bias").unwrap();- let skip_connection_weight = split_tree.leaf_array("skip_connection_weight").unwrap();+ let gate_bias = split_tree.leaf("gate_bias")?.read_allocation()?;+ let skip_connection_weight = split_tree.leaf("skip_connection_weight")?.read_allocation()?;let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)- .expect("Failed to create split in-projection kernel");+ .map_err(MambaMixerError::BackendError)?;let conv_scan =<B::Kernels as Kernels>::Conv1dScanKernel::new(context, data_type, mamba_config.conv_config.has_biases)- .expect("Failed to create conv scan kernel");+ .map_err(MambaMixerError::BackendError)?;let conv_decode = <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context,data_type,mamba_config.conv_config.has_biases,true,)- .expect("Failed to create conv decode kernel");+ .map_err(MambaMixerError::BackendError)?;let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)- .expect("Failed to create conv pack kernel");- let ssd_prefill = SSDPrefillKernels::new(context, data_type).expect("Failed to create SSD prefill kernel");+ .map_err(MambaMixerError::BackendError)?;+ let ssd_prefill = SSDPrefillKernels::new(context, data_type).map_err(MambaMixerError::BackendError)?;let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type, true)- .expect("Failed to create SSD decode kernel");+ .map_err(MambaMixerError::BackendError)?;let prefill_mode = resolve_prefill_mode_from_env();- Self {- layer_index,- config: mamba_config,- in_projection,- out_projection,- split_inproj,- conv_decode,- conv_pack,- conv_scan,- ssd_prefill,- ssd_update,- conv_weight,- conv_bias,- gate_bias,- skip_connection_weight,- prefill_mode,- }+ Ok((+ Self {+ config: mamba_config,+ in_projection,+ out_projection,+ split_inproj,+ conv_decode,+ conv_pack,+ conv_scan,+ ssd_prefill,+ ssd_update,+ conv_weight,+ conv_bias,+ gate_bias,+ skip_connection_weight,+ prefill_mode,+ data_type,+ },+ in_projection_input_hadamard_factors,+ ))}fn run_split_inproj(&self,- state: &mut ForwardPassState<B>,- encoder: &mut Encoder<B>,+ in_proj: &Allocation<B>,suffix_length: usize,- ) {- let in_proj = state.array(ArrayId::SsmInProj);- let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));- let gate = state.array(ArrayId::SsmZ(self.layer_index));- let dt = state.array(ArrayId::SsmDt(self.layer_index));-+ encoder: &mut Encoder<B>,+ ) -> Result<SplitInProjectionOutput<B>, B::Error> {let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let num_heads = self.config.num_heads;let total_dim = conv_dim + inner_dim + num_heads;-+ let mut conv_inputs = encoder.allocate_scratch(size_for_shape(&[suffix_length, conv_dim], self.data_type))?;+ let mut gate = encoder+ .allocate_scratch(size_for_shape(&[suffix_length, num_heads, self.config.head_dim], self.data_type))?;+ let mut time_step = encoder.allocate_scratch(size_for_shape(&[suffix_length, num_heads], self.data_type))?;self.split_inproj.encode(- in_proj.buffer().borrow().deref(),- conv_inputs.buffer().borrow_mut().deref_mut(),- gate.buffer().borrow_mut().deref_mut(),- dt.buffer().borrow_mut().deref_mut(),- self.gate_bias.buffer().borrow().deref(),+ in_proj,+ &mut conv_inputs,+ &mut gate,+ &mut time_step,+ &self.gate_bias,suffix_length as u32,total_dim as u32,conv_dim as u32,inner_dim as u32,num_heads as u32,encoder,- )+ );+ Ok(SplitInProjectionOutput {+ conv_inputs,+ gate,+ time_step,+ })}fn run_conv_scan(&self,- state: &mut ForwardPassState<B>,+ layer: &mut SSMLayer<B>,+ conv_inputs: &Allocation<B>,encoder: &mut Encoder<B>,suffix_length: usize,- ) {- let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));- let conv_state = state.array(ArrayId::SsmConvState(self.layer_index));- let x_arr = state.array(ArrayId::SsmX(self.layer_index));- let b_arr = state.array(ArrayId::SsmB(self.layer_index));- let c_arr = state.array(ArrayId::SsmC(self.layer_index));-- let weight_buf_rc = self.conv_weight.buffer();- let weight_buf_borrow = weight_buf_rc.borrow();- let bias_buf_rc = self.conv_bias.as_ref().map(|arr| arr.buffer());- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());-+ ) -> Result<ConvScanOutput<B>, B::Error> {let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let proj_dim = self.config.num_groups * self.config.state_dim;let state_stride = self.config.kernel_size.saturating_sub(1);+ let mut conv_x = encoder.allocate_scratch(size_for_shape(+ &[suffix_length, self.config.num_heads, self.config.head_dim],+ self.data_type,+ ))?;+ let state_shape = [suffix_length, self.config.num_groups, self.config.state_dim];+ let mut state_b = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;+ let mut state_c = encoder.allocate_scratch(size_for_shape(&state_shape, self.data_type))?;if suffix_length == 1 {if conv_dim > 0 && self.config.kernel_size > 0 {+ let next_state = layer.conv_state.as_mut().unwrap_or(&mut layer.ssm_state);self.conv_decode.encode(- conv_inputs.buffer().borrow().deref(),- weight_buf_borrow.deref(),- bias_buf_borrow.as_deref(),- None::<&B::DenseBuffer>,- x_arr.buffer().borrow_mut().deref_mut(),- b_arr.buffer().borrow_mut().deref_mut(),- c_arr.buffer().borrow_mut().deref_mut(),- conv_state.buffer().borrow_mut().deref_mut(),+ conv_inputs,+ &self.conv_weight,+ self.conv_bias.as_ref(),+ None::<&Allocation<B>>,+ &mut conv_x,+ &mut state_b,+ &mut state_c,+ next_state,self.config.kernel_size as u32,conv_dim as u32,state_stride as u32,@@ -219,39 +243,39 @@ impl<B: Backend> MambaMixer<B> {);}} else {- let padded_buf = if state_stride > 0 {- let array = state.conv_padded_buffer().expect("Missing conv padded buffer");- let buffer = array.buffer();-+ let mut padded = (conv_dim > 0 && state_stride > 0)+ .then(|| {+ encoder.allocate_scratch(size_for_shape(&[suffix_length + state_stride, conv_dim], self.data_type))+ })+ .transpose()?;+ if let Some(padded) = padded.as_mut() {+ let state_in = layer.conv_state.as_ref().unwrap_or(conv_inputs);self.conv_pack.encode(- conv_state.buffer().borrow().deref(),- conv_inputs.buffer().borrow().deref(),- buffer.borrow_mut().deref_mut(),+ state_in,+ conv_inputs,+ padded,state_stride as u32,conv_dim as u32,suffix_length as u32,conv_dim as u32,encoder,);-- Some(buffer)- } else {- None- };+ }if conv_dim > 0 && self.config.kernel_size > 0 {- let padded_borrow = padded_buf.as_ref().map(|b| b.borrow());- let conv_inputs_borrow = conv_inputs.buffer();- let conv_inputs_ref = conv_inputs_borrow.borrow();- let conv_source: &B::DenseBuffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());+ let conv_source = match &padded {+ Some(padded) => padded,+ None => conv_inputs,+ };+ let next_state = layer.conv_state.as_mut().unwrap_or(&mut layer.ssm_state);self.conv_scan.encode(conv_source,- weight_buf_borrow.deref(),- bias_buf_borrow.as_deref(),- x_arr.buffer().borrow_mut().deref_mut(),- b_arr.buffer().borrow_mut().deref_mut(),- c_arr.buffer().borrow_mut().deref_mut(),- conv_state.buffer().borrow_mut().deref_mut(),+ &self.conv_weight,+ self.conv_bias.as_ref(),+ &mut conv_x,+ &mut state_b,+ &mut state_c,+ next_state,suffix_length as u32,self.config.kernel_size as u32,conv_dim as u32,@@ -261,36 +285,40 @@ impl<B: Backend> MambaMixer<B> {proj_dim as u32,self.config.activation.act_type(),encoder,- )+ );}}+ Ok(ConvScanOutput {+ conv_x,+ state_b,+ state_c,+ })}fn run_prefill_ssm(&self,- state: &mut ForwardPassState<B>,+ layer: &mut SSMLayer<B>,+ conv_x: &Allocation<B>,+ state_b: &Allocation<B>,+ state_c: &Allocation<B>,+ time_step: &Allocation<B>,+ gate: &Allocation<B>,encoder: &mut Encoder<B>,suffix_length: usize,- ) {- let x = state.array(ArrayId::SsmX(self.layer_index));- let b = state.array(ArrayId::SsmB(self.layer_index));- let c = state.array(ArrayId::SsmC(self.layer_index));- let dt = state.array(ArrayId::SsmDt(self.layer_index));- let z = state.array(ArrayId::SsmZ(self.layer_index));- let state_arr = state.array(ArrayId::SsmState(self.layer_index));- let out = state.array(ArrayId::AttentionOutput);-+ ) -> Result<Allocation<B>, B::Error> {+ let mut out =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;self.ssd_prefill.encode(encoder,SSDPrefillArguments {- x: x.buffer().borrow().deref(),- dt: dt.buffer().borrow().deref(),- b: b.buffer().borrow().deref(),- c: c.buffer().borrow().deref(),- d: self.skip_connection_weight.buffer().borrow().deref(),- z: z.buffer().borrow().deref(),- state: state_arr.buffer().borrow_mut().deref_mut(),- y: out.buffer().borrow_mut().deref_mut(),+ x: conv_x,+ dt: time_step,+ b: state_b,+ c: state_c,+ d: &self.skip_connection_weight,+ z: gate,+ state: &mut layer.ssm_state,+ y: &mut out,suffix_len: suffix_length,group_size: (self.config.num_heads / self.config.num_groups) as u32,state_size: self.config.state_dim as u32,@@ -302,23 +330,23 @@ impl<B: Backend> MambaMixer<B> {head_dim: self.config.head_dim,},self.prefill_mode,- )+ );+ Ok(out)}fn run_decode_ssm(&self,- state: &mut ForwardPassState<B>,+ layer: &mut SSMLayer<B>,+ conv_x: &Allocation<B>,+ state_b: &Allocation<B>,+ state_c: &Allocation<B>,+ time_step: &Allocation<B>,+ gate: &Allocation<B>,encoder: &mut Encoder<B>,suffix_length: usize,- ) {- let x = state.array(ArrayId::SsmX(self.layer_index));- let b = state.array(ArrayId::SsmB(self.layer_index));- let c = state.array(ArrayId::SsmC(self.layer_index));- let dt = state.array(ArrayId::SsmDt(self.layer_index));- let z = state.array(ArrayId::SsmZ(self.layer_index));- let state_arr = state.array(ArrayId::SsmState(self.layer_index));- let y = state.array(ArrayId::AttentionOutput);-+ ) -> Result<Allocation<B>, B::Error> {+ let mut out =+ encoder.allocate_scratch(size_for_shape(&[suffix_length, self.config.inner_dim()], self.data_type))?;let h = self.config.num_heads as u32;let g = self.config.num_groups as u32;let dh = self.config.head_dim as u32;@@ -332,15 +360,15 @@ impl<B: Backend> MambaMixer<B> {let state_size = n as i32;self.ssd_update.encode(- x.buffer().borrow().deref(),- dt.buffer().borrow().deref(),- b.buffer().borrow().deref(),- c.buffer().borrow().deref(),- self.skip_connection_weight.buffer().borrow().deref(),- z.buffer().borrow().deref(),- None::<&B::DenseBuffer>,- y.buffer().borrow_mut().deref_mut(),- state_arr.buffer().borrow_mut().deref_mut(),+ conv_x,+ time_step,+ state_b,+ state_c,+ &self.skip_connection_weight,+ gate,+ None::<&Allocation<B>>,+ &mut out,+ &mut layer.ssm_state,group_size as u32,state_size as u32,&x_strides,@@ -352,27 +380,40 @@ impl<B: Backend> MambaMixer<B> {dh,encoder,);+ Ok(out)}pub(crate) fn encode(&self,- state: &mut ForwardPassState<B>,+ args: MambaArguments<B>,+ input: Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let active_row_count = state.active_row_count();- if active_row_count == 0 {- return Ok(());- }+ ) -> Result<Allocation<B>, B::Error> {+ let MambaArguments {+ active_row_count,+ layer,+ } = args;+ assert!(active_row_count > 0, "Mamba mixer requires at least one active row");- self.in_projection.encode(state, encoder)?;- self.run_split_inproj(state, encoder, active_row_count);- self.run_conv_scan(state, encoder, active_row_count);+ let in_proj = self.in_projection.encode(input, active_row_count, encoder)?;+ let SplitInProjectionOutput {+ conv_inputs,+ gate,+ time_step,+ } = self.run_split_inproj(&in_proj, active_row_count, encoder)?;+ let ConvScanOutput {+ conv_x,+ state_b,+ state_c,+ } = self.run_conv_scan(layer, &conv_inputs, encoder, active_row_count)?;if active_row_count == 1 {- self.run_decode_ssm(state, encoder, active_row_count);+ let ssm_output =+ self.run_decode_ssm(layer, &conv_x, &state_b, &state_c, &time_step, &gate, encoder, active_row_count)?;+ self.out_projection.encode(ssm_output, active_row_count, encoder)} else {- self.run_prefill_ssm(state, encoder, active_row_count);+ let ssm_output =+ self.run_prefill_ssm(layer, &conv_x, &state_b, &state_c, &time_step, &gate, encoder, active_row_count)?;+ self.out_projection.encode(ssm_output, active_row_count, encoder)}- self.out_projection.encode(state, encoder)?;- Ok(())}}diff --git a/crates/backend-uzu/src/encodable_block/mlp/dense.rs b/crates/backend-uzu/src/encodable_block/mlp/dense.rsindex 2d6558228..00449dea4 100644--- a/crates/backend-uzu/src/encodable_block/mlp/dense.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/dense.rs@@ -1,17 +1,18 @@//! MLP block encodable.-use std::ops::{Deref, DerefMut};-use super::{super::linear::Linear, Mlp};use crate::{- backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},- forward_pass::state::{ArrayId, ForwardPassState},+ DataType,+ array::size_for_shape,+ backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},};pub struct DenseMlp<B: Backend> {up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,+ hidden_dim: usize,+ data_type: DataType,}impl<B: Backend> DenseMlp<B> {@@ -19,11 +20,15 @@ impl<B: Backend> DenseMlp<B> {up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,down: Box<dyn Linear<B>>,+ hidden_dim: usize,+ data_type: DataType,) -> Self {Self {up,gate,down,+ hidden_dim,+ data_type,}}}@@ -31,22 +36,13 @@ impl<B: Backend> DenseMlp<B> {impl<B: Backend> Mlp<B> for DenseMlp<B> {fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- // Up- self.up.encode(state, encoder)?;-- // Gate act+mul (fused_up -> hidden)- {- let fused = state.array(ArrayId::MlpFusedUp);- let hidden = state.array(ArrayId::MlpHidden);- let m = fused.shape()[0] as i32;- self.gate.encode(encoder, fused.buffer().borrow().deref(), hidden.buffer().borrow_mut().deref_mut(), m)?;- }-- // Down- self.down.encode(state, encoder)?;- Ok(())+ ) -> Result<Allocation<B>, B::Error> {+ let fused_up = self.up.encode(input, batch_dim, encoder)?;+ let mut hidden = encoder.allocate_scratch(size_for_shape(&[batch_dim, self.hidden_dim], self.data_type))?;+ self.gate.encode(encoder, &fused_up, &mut hidden, batch_dim as i32)?;+ self.down.encode(hidden, batch_dim, encoder)}}diff --git a/crates/backend-uzu/src/encodable_block/mlp/mod.rs b/crates/backend-uzu/src/encodable_block/mlp/mod.rsindex 61e91adf0..b301563c3 100644--- a/crates/backend-uzu/src/encodable_block/mlp/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/mod.rs@@ -8,18 +8,18 @@ use thiserror::Error;use super::linear::{Linear, LinearBlockError};use crate::{DataType,- backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},+ backends::common::{Allocation, Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},config::MLPConfig,- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};pub trait Mlp<B: Backend> {fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error>;+ ) -> Result<Allocation<B>, B::Error>;}#[derive(Debug, Error)]@@ -41,30 +41,24 @@ impl<B: Backend> dyn Mlp<B> {hidden_dimension: usize,context: &B::Context,parameter_tree: &ParameterTree<B::Context>,- ) -> Result<(Box<dyn Mlp<B>>, Option<B::DenseBuffer>), MlpBlockError<B>> {+ ) -> Result<(Box<dyn Mlp<B>>, Option<Allocation<B>>), MlpBlockError<B>> {if let MLPConfig::Dense(dense_config) = config {let data_type: DataType = dense_config.linear_config.activation_precision().into();let (up_projection, up_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&dense_config.linear_config,- false,model_dimension,[2 * hidden_dimension],context,¶meter_tree.subtree("up_projection")?,- ArrayId::Main,- ArrayId::MlpFusedUp,)?;let (down_projection, down_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&dense_config.linear_config,- false,hidden_dimension,[model_dimension],context,¶meter_tree.subtree("down_projection")?,- ArrayId::MlpHidden,- ArrayId::Main,)?;let gate = MlpGateActMulEncodable::new(@@ -76,7 +70,10 @@ impl<B: Backend> dyn Mlp<B> {).map_err(MlpBlockError::BackendError)?;- return Ok((Box::new(DenseMlp::new(up_projection, gate, down_projection)), up_input_hadamard_factors));+ return Ok((+ Box::new(DenseMlp::new(up_projection, gate, down_projection, hidden_dimension, data_type)),+ up_input_hadamard_factors,+ ));}if let MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {diff --git a/crates/backend-uzu/src/encodable_block/mlp/moe.rs b/crates/backend-uzu/src/encodable_block/mlp/moe.rsindex ef9f50c49..c85d49635 100644--- a/crates/backend-uzu/src/encodable_block/mlp/moe.rs+++ b/crates/backend-uzu/src/encodable_block/mlp/moe.rs@@ -1,17 +1,12 @@//! MoE (Mixture of Experts) block encodable.-use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};-use thiserror::Error;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,gpu_types::ActivationType,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,@@ -24,21 +19,19 @@ use crate::{},config::{LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig},encodable_block::mlp::Mlp,- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};struct RouterBlock<B: Backend> {- weights_buf: Rc<RefCell<B::DenseBuffer>>,- biases_buf: Rc<RefCell<B::DenseBuffer>>,+ weights: Allocation<B>,+ biases: Allocation<B>,}-#[derive(Clone)]struct SharedMoeWeights<B: Backend> {- pub w13_buf: Rc<RefCell<B::DenseBuffer>>,- pub w2_buf: Rc<RefCell<B::DenseBuffer>>,- pub up_biases_buf: Rc<RefCell<B::DenseBuffer>>,- pub down_biases_buf: Rc<RefCell<B::DenseBuffer>>,+ pub w13: Allocation<B>,+ pub w2: Allocation<B>,+ pub up_biases: Allocation<B>,+ pub down_biases: Allocation<B>,}pub struct MoeBlock<B: Backend> {@@ -64,7 +57,9 @@ pub enum MoeBlockError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Parameter loader error: {0}")]- ParameterLoaderError(#[source] ParameterLoaderError<B>),+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+ #[error("MoE num_active_routed_experts must be greater than 0")]+ InvalidActiveExpertCount,}impl<B: Backend> MoeBlock<B> {@@ -75,6 +70,9 @@ impl<B: Backend> MoeBlock<B> {hidden_dim: usize,parameter_tree: &ParameterTree<B::Context>,) -> Result<Self, MoeBlockError<B>> {+ if moe_config.num_active_routed_experts == 0 {+ return Err(MoeBlockError::InvalidActiveExpertCount);+ }let data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();let router_data_type: DataType = moe_config.router_config.activation_precision().into();@@ -93,11 +91,13 @@ impl<B: Backend> MoeBlock<B> {LinearConfig::FullPrecision {..} => {- let weights_arr = router_tree.leaf_array("weights").map_err(MoeBlockError::ParameterLoaderError)?;- let biases_arr = router_tree.leaf_array("biases").map_err(MoeBlockError::ParameterLoaderError)?;+ let weights =+ router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?.read_allocation()?;+ let biases =+ router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?.read_allocation()?;RouterBlock {- weights_buf: weights_arr.buffer(),- biases_buf: biases_arr.buffer(),+ weights,+ biases,}},LinearConfig::QLoRA {@@ -131,35 +131,39 @@ impl<B: Backend> MoeBlock<B> {let experts_tree = parameter_tree.subtree("experts").map_err(MoeBlockError::ParameterLoaderError)?;- let w13_arr = experts_tree+ let w13 = experts_tree.subtree("up_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf_array("weights")- .map_err(MoeBlockError::ParameterLoaderError)?;+ .leaf("weights")+ .map_err(MoeBlockError::ParameterLoaderError)?+ .read_allocation()?;- let w2_arr = experts_tree+ let w2 = experts_tree.subtree("down_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf_array("weights")- .map_err(MoeBlockError::ParameterLoaderError)?;+ .leaf("weights")+ .map_err(MoeBlockError::ParameterLoaderError)?+ .read_allocation()?;- let up_biases_arr = experts_tree+ let up_biases = experts_tree.subtree("up_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf_array("biases")- .map_err(MoeBlockError::ParameterLoaderError)?;+ .leaf("biases")+ .map_err(MoeBlockError::ParameterLoaderError)?+ .read_allocation()?;- let down_biases_arr = experts_tree+ let down_biases = experts_tree.subtree("down_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf_array("biases")- .map_err(MoeBlockError::ParameterLoaderError)?;+ .leaf("biases")+ .map_err(MoeBlockError::ParameterLoaderError)?+ .read_allocation()?;let shared_weights = SharedMoeWeights {- w13_buf: w13_arr.buffer(),- w2_buf: w2_arr.buffer(),- up_biases_buf: up_biases_arr.buffer(),- down_biases_buf: down_biases_arr.buffer(),+ w13,+ w2,+ up_biases,+ down_biases,};Ok(Self {@@ -197,68 +201,21 @@ impl<B: Backend> MoeBlock<B> {impl<B: Backend> Mlp<B> for MoeBlock<B> {fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let suffix_length = state.active_row_count();-- let main_buf = state.array(ArrayId::Main).buffer();- let topk_ids_buf = state.array(ArrayId::MoeTopkIds).buffer();- let topk_probs_buf = state.array(ArrayId::MoeTopkProbs).buffer();- let offsets_buf = state.array(ArrayId::MoeOffsets).buffer();- let sumk_buf = state.array(ArrayId::MoeSumK).buffer();- let bucketed_ids_buf = state.array(ArrayId::MoeBucketedTokenIds).buffer();- let bucketed_probs_buf = state.array(ArrayId::MoeBucketedProbs).buffer();- let x_perm_buf = state.array(ArrayId::MoeXPerm).buffer();- let tok2row_buf = state.array(ArrayId::MoeTok2Row).buffer();- let y_partial_buf = state.array(ArrayId::MoeYPartial).buffer();- let tile_counts_buf = state.array(ArrayId::MoeTileCounts).buffer();- let tile_offsets_buf = state.array(ArrayId::MoeTileOffsets).buffer();- let tile_map_buf = state.array(ArrayId::MoeTileMap).buffer();- let total_tiles_buf = state.array(ArrayId::MoeTotalTiles).buffer();- let dispatch_args_buf = state.array(ArrayId::MoeDispatchArgs).buffer();- let partials_buf = state.array(ArrayId::MoeScatterPartials).buffer();- let block_bases_buf = state.array(ArrayId::MoeScatterBlockBases).buffer();- let block_alloc_buf = state.array(ArrayId::MoeBlockAlloc).buffer();- let hidden_buf = state.array(ArrayId::MoeHidden).buffer();- let row_expert_map_buf = state.array(ArrayId::MoeTwoPassRowExpertMap).buffer();+ ) -> Result<Allocation<B>, B::Error> {+ let suffix_length = batch_dim;+ if suffix_length == 0 {+ return Ok(input);+ }let e = self.moe_config.num_routed_experts;let k = self.moe_config.num_active_routed_experts;let k_tile = 128;-- // Clear internal MoE buffers- if suffix_length > 0 && k > 0 {- let entries = suffix_length * k;- let topk_bytes = entries * std::mem::size_of::<u32>();- let tok2row_bytes = entries * std::mem::size_of::<i32>();-- // Clear topk_ids and tok2row buffers- if topk_bytes > 0 {- encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);- }- if tok2row_bytes > 0 {- encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);- }-- // Clear hidden buffer- let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();- if hidden_bytes > 0 {- encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);- }-- // Clear y_partial buffer- let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();- if y_partial_bytes > 0 {- encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);- }-- // Clear x_perm buffer- let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();- if x_perm_bytes > 0 {- encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);- }- }+ let total_rows = suffix_length * k;+ let num_blocks = suffix_length.div_ceil(256).max(1);+ let num_tiles = e.div_ceil(512).max(1);// Use fused Router+TopK kernel for non-quantized routersif self.model_dim % 4 != 0 {@@ -271,14 +228,20 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {panic!("MoE fused router+bottomk failed: {k} > 128");}+ let mut topk_ids = encoder.allocate_scratch(size_for_shape(&[suffix_length, k], DataType::I32))?;+ let mut topk_probs = encoder.allocate_scratch(size_for_shape(&[suffix_length, k], self.data_type))?;++ if suffix_length > 0 && k > 0 {+ encoder.encode_fill(&mut topk_ids, 0xFF);+ }+if suffix_length > 0 && e > 0 && k > 0 {- // Use the fused router+topk kernelself.router_topk_kernel.encode(- main_buf.borrow().deref(),- self.router.weights_buf.borrow().deref(),- self.router.biases_buf.borrow().deref(),- topk_ids_buf.borrow_mut().deref_mut(),- topk_probs_buf.borrow_mut().deref_mut(),+ &input,+ &self.router.weights,+ &self.router.biases,+ &mut topk_ids,+ &mut topk_probs,suffix_length as u32,self.model_dim as u32,e as u32,@@ -288,23 +251,35 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {);}+ let mut offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;+ let mut sumk = encoder.allocate_scratch(size_for_shape(&[1], DataType::U32))?;+ let scatter_entries = num_blocks * num_tiles * 512;+ let mut partials = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;self.counts_offsets_kernel.encode(- topk_ids_buf.borrow().deref(),- offsets_buf.borrow_mut().deref_mut(),- sumk_buf.borrow_mut().deref_mut(),- partials_buf.borrow_mut().deref_mut(),+ &topk_ids,+ &mut offsets,+ &mut sumk,+ &mut partials,suffix_length as u32,e as u32,k as u32,encoder,);- let num_blocks = ((suffix_length + 255) / 256).max(1);- let num_tiles = ((e + 512 - 1) / 512).max(1);+ let mut block_bases = encoder.allocate_scratch(size_for_shape(&[scatter_entries], DataType::U32))?;+ let mut block_alloc = encoder.allocate_scratch(size_for_shape(&[num_blocks * num_tiles], DataType::U32))?;+ let mut bucketed_ids = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;+ let mut bucketed_probs = encoder.allocate_scratch(size_for_shape(&[total_rows], self.data_type))?;+ let mut tok2row = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::I32))?;++ if suffix_length > 0 && k > 0 {+ encoder.encode_fill(&mut tok2row, 0xFF);+ }+self.scatter_bases_kernel.encode(- partials_buf.borrow().deref(),- block_bases_buf.borrow_mut().deref_mut(),- block_alloc_buf.borrow_mut().deref_mut(),+ &partials,+ &mut block_bases,+ &mut block_alloc,e as u32,num_blocks as u32,num_tiles as u32,@@ -312,30 +287,35 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {encoder,);self.scatter_map_kernel.encode(- topk_ids_buf.borrow().deref(),- topk_probs_buf.borrow().deref(),- offsets_buf.borrow().deref(),- block_bases_buf.borrow().deref(),- block_alloc_buf.borrow().deref(),- bucketed_ids_buf.borrow_mut().deref_mut(),- bucketed_probs_buf.borrow_mut().deref_mut(),+ &topk_ids,+ &topk_probs,+ &offsets,+ &block_bases,+ &block_alloc,+ &mut bucketed_ids,+ &mut bucketed_probs,suffix_length as u32,e as u32,k as u32,num_blocks as u32,num_tiles as u32,- tok2row_buf.borrow_mut().deref_mut(),+ &mut tok2row,encoder,);+ let mut x_perm = encoder.allocate_scratch(size_for_shape(&[total_rows, self.model_dim], self.data_type))?;+ if suffix_length > 0 && k > 0 {+ encoder.encode_fill(&mut x_perm, 0);+ }+self.gather_kernels.encode(encoder,self.data_type,MoeGatherArguments {- x_buffer: main_buf.borrow().deref(),- bucketed_ids_buffer: bucketed_ids_buf.borrow().deref(),- x_perm_buffer: x_perm_buf.borrow_mut().deref_mut(),- sumk_buffer: sumk_buf.borrow().deref(),+ x: &input,+ bucketed_ids: &bucketed_ids,+ x_perm: &mut x_perm,+ sumk: &sumk,t: suffix_length,k,d_model: self.model_dim,@@ -351,27 +331,33 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {let up_clip_max = self.moe_config.expert_config.up_clipping[1];let silu_alpha = self.moe_config.expert_config.activation.alpha();- if suffix_length == 1 {- let total_rows = suffix_length * k;+ let mut row_expert_map = encoder.allocate_scratch(size_for_shape(&[total_rows], DataType::U32))?;+ let mut tile_counts = encoder.allocate_scratch(size_for_shape(&[e], DataType::U32))?;+ let mut tile_offsets = encoder.allocate_scratch(size_for_shape(&[e + 1], DataType::U32))?;+ let h_blocks = self.hidden_dim.div_ceil(4);+ let mut tile_map =+ encoder.allocate_scratch(size_for_shape(&[total_rows * h_blocks.max(1) * 3], DataType::U32))?;+ let mut total_tiles = encoder.allocate_scratch(size_for_shape(&[8], DataType::U32))?;+ let mut dispatch_args = encoder.allocate_scratch(size_for_shape(&[3], DataType::U32))?;++ let y_partial = if suffix_length == 1 {let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;self.experts_two_pass_decode_kernel.encode(encoder,MoeExpertsTwoPassArguments {- x_perm_buffer: x_perm_buf.borrow().deref(),- expert_offsets: offsets_buf.borrow().deref(),- row_expert_map: row_expert_map_buf.borrow_mut().deref_mut(),- hidden_buffer: hidden_buf.borrow_mut().deref_mut(),- output_buffer: y_partial_buf.borrow_mut().deref_mut(),- w13_all: self.shared_weights.w13_buf.borrow().deref(),- w2_all: self.shared_weights.w2_buf.borrow().deref(),- up_biases: self.shared_weights.up_biases_buf.borrow().deref(),- down_biases: self.shared_weights.down_biases_buf.borrow().deref(),- tile_counts: tile_counts_buf.borrow_mut().deref_mut(),- tile_offsets: tile_offsets_buf.borrow_mut().deref_mut(),- tile_map: tile_map_buf.borrow_mut().deref_mut(),- total_tiles: total_tiles_buf.borrow_mut().deref_mut(),- dispatch_args: dispatch_args_buf.borrow_mut().deref_mut(),+ x_perm: &x_perm,+ expert_offsets: &offsets,+ row_expert_map: &mut row_expert_map,+ w13_all: &self.shared_weights.w13,+ w2_all: &self.shared_weights.w2,+ up_biases: &self.shared_weights.up_biases,+ down_biases: &self.shared_weights.down_biases,+ tile_counts: &mut tile_counts,+ tile_offsets: &mut tile_offsets,+ tile_map: &mut tile_map,+ total_tiles: &mut total_tiles,+ dispatch_args: &mut dispatch_args,total_rows,d_model: self.model_dim,d_ff: self.hidden_dim,@@ -385,41 +371,22 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {silu_alpha,data_type: self.data_type,},- );+ )?} else {- let total_rows = suffix_length * k;let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;-- let x_perm_borrow = x_perm_buf.borrow();- let offsets_borrow = offsets_buf.borrow();- let mut row_expert_map_borrow = row_expert_map_buf.borrow_mut();- let mut hidden_borrow = hidden_buf.borrow_mut();- let mut y_partial_borrow = y_partial_buf.borrow_mut();- let w13_borrow = self.shared_weights.w13_buf.borrow();- let w2_borrow = self.shared_weights.w2_buf.borrow();- let up_biases_borrow = self.shared_weights.up_biases_buf.borrow();- let down_biases_borrow = self.shared_weights.down_biases_buf.borrow();- let mut tile_counts_borrow = tile_counts_buf.borrow_mut();- let mut tile_offsets_borrow = tile_offsets_buf.borrow_mut();- let mut tile_map_borrow = tile_map_buf.borrow_mut();- let mut total_tiles_borrow = total_tiles_buf.borrow_mut();- let mut dispatch_args_borrow = dispatch_args_buf.borrow_mut();-let args = MoeExpertsTwoPassArguments {- x_perm_buffer: x_perm_borrow.deref(),- expert_offsets: offsets_borrow.deref(),- row_expert_map: row_expert_map_borrow.deref_mut(),- hidden_buffer: hidden_borrow.deref_mut(),- output_buffer: y_partial_borrow.deref_mut(),- w13_all: w13_borrow.deref(),- w2_all: w2_borrow.deref(),- up_biases: up_biases_borrow.deref(),- down_biases: down_biases_borrow.deref(),- tile_counts: tile_counts_borrow.deref_mut(),- tile_offsets: tile_offsets_borrow.deref_mut(),- tile_map: tile_map_borrow.deref_mut(),- total_tiles: total_tiles_borrow.deref_mut(),- dispatch_args: dispatch_args_borrow.deref_mut(),+ x_perm: &x_perm,+ expert_offsets: &offsets,+ row_expert_map: &mut row_expert_map,+ w13_all: &self.shared_weights.w13,+ w2_all: &self.shared_weights.w2,+ up_biases: &self.shared_weights.up_biases,+ down_biases: &self.shared_weights.down_biases,+ tile_counts: &mut tile_counts,+ tile_offsets: &mut tile_offsets,+ tile_map: &mut tile_map,+ total_tiles: &mut total_tiles,+ dispatch_args: &mut dispatch_args,total_rows,d_model: self.model_dim,d_ff: self.hidden_dim,@@ -433,21 +400,22 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {silu_alpha,data_type: self.data_type,};- self.experts_two_pass_prefill_kernel.encode(encoder, args);- }+ self.experts_two_pass_prefill_kernel.encode(encoder, args)?+ };+ let mut output = encoder.allocate_scratch(size_for_shape(&[suffix_length, self.model_dim], self.data_type))?;self.finalize_kernel.encode(- tok2row_buf.borrow().deref(),- topk_probs_buf.borrow().deref(),- y_partial_buf.borrow().deref(),- main_buf.borrow_mut().deref_mut(),+ &tok2row,+ &topk_probs,+ &y_partial,+ &mut output,suffix_length as u32,self.model_dim as u32,k as u32,encoder,);- Ok(())+ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/mod.rs b/crates/backend-uzu/src/encodable_block/mod.rsindex 84f5b1b2d..18fd58478 100644--- a/crates/backend-uzu/src/encodable_block/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mod.rs@@ -1,4 +1,3 @@-mod activation;mod attention;mod classifier_layer;mod decoder;@@ -17,19 +16,16 @@ mod rms_norm;mod rope;mod sampling;mod short_conv_mixer;-mod tensor_add_swap;-mod tensor_copy;-pub use activation::Activation;-pub use attention::Attention;+pub use attention::{Attention, AttentionArguments};pub use classifier_layer::ClassifierLayer;-pub use decoder::Decoder;-pub(crate) use delta_net_mixer::DeltaNetMixer;+pub use decoder::{Decoder, DecoderArguments, DecoderDecodeInput};+pub(crate) use delta_net_mixer::{DeltaNetArguments, DeltaNetMixer};pub use embedding::Embedding;-pub use layer::LayerExecutables;+pub use layer::{LayerArguments, LayerExecutables};pub use layer_norm::{LayerNorm, LayerNormError};pub use linear::Linear;-pub(crate) use mamba_mixer::MambaMixer;+pub(crate) use mamba_mixer::{MambaArguments, MambaMixer};pub use mlp::Mlp;pub use normalization::Normalization;pub use pooling::Pooling;@@ -37,10 +33,10 @@ pub use prediction_head::ClassifierPredictionHead;pub use qk_norm::QKNorm;pub use rms_norm::{RMSNorm, RMSNormError};pub use rope::Rope;-pub use sampling::Sampling;+pub(crate) use sampling::SamplingInputs;+pub use sampling::{Sampling, SamplingArguments};+pub(crate) use short_conv_mixer::ShortConvArguments;pub use short_conv_mixer::ShortConvMixer;-pub use tensor_add_swap::TensorAddSwap;-pub use tensor_copy::TensorCopy;#[derive(Clone)]pub struct EncodingParameters {@@ -53,12 +49,4 @@ impl EncodingParameters {projection_step: None,}}-- pub fn with_projection(- mut self,- projection_step: usize,- ) -> Self {- self.projection_step = Some(projection_step);- self- }}diff --git a/crates/backend-uzu/src/encodable_block/normalization.rs b/crates/backend-uzu/src/encodable_block/normalization.rsindex d2349e401..ecc943705 100644--- a/crates/backend-uzu/src/encodable_block/normalization.rs+++ b/crates/backend-uzu/src/encodable_block/normalization.rs@@ -5,9 +5,8 @@ use thiserror::Error;use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};use crate::{DataType,- backends::common::{Backend, Encoder},+ backends::common::{Allocation, Backend, Encoder},config::NormalizationConfig,- forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};@@ -32,29 +31,18 @@ impl<B: Backend> Normalization<B> {context: &B::Context,intermediate_data_type: DataType,config: NormalizationConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,) -> Result<Self, NormalizationError<B>> {if config.subtract_mean {- Ok(Self::LayerNorm(LayerNorm::new(- context,- intermediate_data_type,- config,- input_array_id,- output_array_id,- parameter_tree,- )?))+ Ok(Self::LayerNorm(LayerNorm::new(context, intermediate_data_type, config, parameter_tree)?))} else {Ok(Self::RMSNorm(RMSNorm::new(context,intermediate_data_type,config,- input_array_id,- output_array_id,parameter_tree,None,- None,+ false,false,)?))}@@ -62,12 +50,14 @@ impl<B: Backend> Normalization<B> {pub fn encode(&self,- state: &mut ForwardPassState<B>,+ input: &Allocation<B>,+ row_offset: usize,+ row_count: usize,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {+ ) -> Result<Allocation<B>, B::Error> {match self {- Self::LayerNorm(layer_norm) => layer_norm.encode(state, encoder),- Self::RMSNorm(rms_norm) => rms_norm.encode(state, encoder),+ Self::LayerNorm(layer_norm) => layer_norm.encode(input, row_offset, row_count, encoder),+ Self::RMSNorm(rms_norm) => rms_norm.encode(input, row_offset, row_count, None, encoder),}}}diff --git a/crates/backend-uzu/src/encodable_block/pooling.rs b/crates/backend-uzu/src/encodable_block/pooling.rsindex 82ee17671..ced65d036 100644--- a/crates/backend-uzu/src/encodable_block/pooling.rs+++ b/crates/backend-uzu/src/encodable_block/pooling.rs@@ -1,15 +1,13 @@//! Pooling encodable for sequence-level aggregation.-use std::ops::{Deref, DerefMut};-use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, PoolingClsKernel, PoolingMeanKernel},},config::PoolingType,- forward_pass::state::{ArrayId, ForwardPassState},};enum PoolingKernel<B: Backend> {@@ -20,8 +18,8 @@ enum PoolingKernel<B: Backend> {impl<B: Backend> PoolingKernel<B> {fn encode(&self,- input: &B::DenseBuffer,- output: &mut B::DenseBuffer,+ input: &Allocation<B>,+ output: &mut Allocation<B>,seq_len: u32,hidden_dim: u32,batch_dim: u32,@@ -37,6 +35,7 @@ impl<B: Backend> PoolingKernel<B> {pub struct Pooling<B: Backend> {pooling_kernel: PoolingKernel<B>,model_dim: usize,+ data_type: DataType,}impl<B: Backend> Pooling<B> {@@ -55,36 +54,20 @@ impl<B: Backend> Pooling<B> {Ok(Self {pooling_kernel,model_dim,+ data_type,})}pub fn encode(&self,- state: &mut ForwardPassState<B>,+ seq_len: usize,+ main: &Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {+ ) -> Result<Allocation<B>, B::Error> {let batch_dim = 1;- let seq_len = state.aux_buffers_suffix_length();-- let main_array = state.array(ArrayId::Main);- let pooling_array = state.array(ArrayId::ClassifierPooling);-- self.pooling_kernel.encode(- main_array.buffer().borrow().deref(),- pooling_array.buffer().borrow_mut().deref_mut(),- seq_len as u32,- self.model_dim as u32,- batch_dim,- encoder,- );-- #[cfg(feature = "tracing")]- {- let output_pooling_trace = state.traces().borrow().output_pooling().clone();- state.encode_copy_array(encoder, ArrayId::ClassifierPooling, output_pooling_trace);- }+ let mut pooling = encoder.allocate_scratch(size_for_shape(&[1, self.model_dim], self.data_type))?;- state.set_active_row_count(pooling_array.shape()[0]);- Ok(())+ self.pooling_kernel.encode(main, &mut pooling, seq_len as u32, self.model_dim as u32, batch_dim, encoder);+ Ok(pooling)}}diff --git a/crates/backend-uzu/src/encodable_block/prediction_head.rs b/crates/backend-uzu/src/encodable_block/prediction_head.rsindex aca2827c7..c918b2c3f 100644--- a/crates/backend-uzu/src/encodable_block/prediction_head.rs+++ b/crates/backend-uzu/src/encodable_block/prediction_head.rs@@ -1,55 +1,67 @@//! Prediction head encodable for classification output.-#[cfg(feature = "tracing")]-use crate::forward_pass::state::ArrayId;use crate::{- backends::common::{Backend, Encoder},- encodable_block::{Activation, Normalization, linear::Linear},- forward_pass::state::ForwardPassState,+ DataType,+ array::size_for_shape,+ backends::common::{+ ActivationConfig, Allocation, Backend, Encoder,+ gpu_types::ActivationType,+ kernel::{ActivationKernel, Kernels},+ },+ encodable_block::{Normalization, linear::Linear},};pub struct ClassifierPredictionHead<B: Backend> {dense: Box<dyn Linear<B>>,- activation: Activation<B>,+ activation_kernel: <B::Kernels as Kernels>::ActivationKernel,+ activation: ActivationConfig,+ activation_data_type: DataType,norm: Normalization<B>,readout: Box<dyn Linear<B>>,- #[allow(dead_code)]- num_labels: usize,+ hidden_dim: usize,}impl<B: Backend> ClassifierPredictionHead<B> {pub fn new(+ context: &B::Context,dense: Box<dyn Linear<B>>,- activation: Activation<B>,+ activation: ActivationConfig,+ activation_data_type: DataType,norm: Normalization<B>,readout: Box<dyn Linear<B>>,- num_labels: usize,- ) -> Self {- Self {+ hidden_dim: usize,+ ) -> Result<Self, B::Error> {+ let activation_kernel = <B::Kernels as Kernels>::ActivationKernel::new(context, activation_data_type, false)?;+ Ok(Self {dense,+ activation_kernel,activation,+ activation_data_type,norm,readout,- num_labels,- }+ hidden_dim,+ })}pub fn encode(&self,- state: &mut ForwardPassState<B>,+ input: Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- self.dense.encode(state, encoder)?;- self.activation.encode(state, encoder)?;- self.norm.encode(state, encoder)?;- self.readout.encode(state, encoder)?;-- #[cfg(feature = "tracing")]- {- let traces = state.traces().clone();- state.encode_copy_array(encoder, ArrayId::ClassifierPredictionHeadLogits, traces.borrow().logits.clone());+ ) -> Result<Allocation<B>, B::Error> {+ let batch_dim = 1;+ let dense = self.dense.encode(input, batch_dim, encoder)?;+ if self.activation.act_type() == ActivationType::IDENTITY {+ panic!("Identity activation is not supported for kernel");}-- Ok(())+ let mut activated = encoder.allocate_scratch(size_for_shape(&[self.hidden_dim], self.activation_data_type))?;+ self.activation_kernel.encode(+ Some(&dense),+ &mut activated,+ self.hidden_dim as u32,+ self.activation.act_type(),+ encoder,+ );+ let normalized = self.norm.encode(&activated, 0, batch_dim, encoder)?;+ self.readout.encode(normalized, batch_dim, encoder)}}diff --git a/crates/backend-uzu/src/encodable_block/qk_norm.rs b/crates/backend-uzu/src/encodable_block/qk_norm.rsindex cbda452d7..a4c7b18c9 100644--- a/crates/backend-uzu/src/encodable_block/qk_norm.rs+++ b/crates/backend-uzu/src/encodable_block/qk_norm.rs@@ -1,21 +1,14 @@//! QK Normalization encodable.-use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};-use thiserror::Error;use crate::{DataType,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, QKNormKernel},},config::{NormalizationConfig, UpcastMode},- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};@@ -32,9 +25,8 @@ pub struct QKNorm<B: Backend> {key_kernel: Option<<B::Kernels as Kernels>::QKNormKernel>,query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,- qkv_array_id: ArrayId,- query_scales_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,- key_scales_buffer: Option<Rc<RefCell<B::DenseBuffer>>>,+ query_scales: Option<Allocation<B>>,+ key_scales: Option<Allocation<B>>,num_q_heads: usize,num_kv_heads: usize,head_dim: usize,@@ -46,7 +38,6 @@ impl<B: Backend> QKNorm<B> {intermediate_data_type: DataType,query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,- qkv_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,num_q_heads: usize,num_kv_heads: usize,@@ -54,12 +45,16 @@ impl<B: Backend> QKNorm<B> {) -> Result<Self, QKNormError<B>> {let mut query_kernel = None;let mut key_kernel = None;- let mut query_scales_buffer = None;- let mut key_scales_buffer = None;+ let mut query_scales = None;+ let mut key_scales = None;// Setup query normalization if configuredif let Some(ref q_config) = query_config {- let scales = parameter_tree.leaf_array("query_norm.scales").map_err(QKNormError::ParameterError)?;+ let scales = parameter_tree+ .leaf("query_norm.scales")+ .map_err(QKNormError::ParameterError)?+ .read_allocation()+ .map_err(QKNormError::ParameterError)?;let accumulation_data_type: DataType = q_config.accumulation_precision.into();let scale_data_type: DataType = q_config.scale_precision.into();@@ -79,12 +74,16 @@ impl<B: Backend> QKNorm<B> {.map_err(QKNormError::BackendError)?;query_kernel = Some(kernel);- query_scales_buffer = Some(scales.buffer());+ query_scales = Some(scales);}// Setup key normalization if configuredif let Some(ref k_config) = key_config {- let scales = parameter_tree.leaf_array("key_norm.scales").map_err(QKNormError::ParameterError)?;+ let scales = parameter_tree+ .leaf("key_norm.scales")+ .map_err(QKNormError::ParameterError)?+ .read_allocation()+ .map_err(QKNormError::ParameterError)?;let accumulation_data_type: DataType = k_config.accumulation_precision.into();let scale_data_type: DataType = k_config.scale_precision.into();@@ -104,7 +103,7 @@ impl<B: Backend> QKNorm<B> {.map_err(QKNormError::BackendError)?;key_kernel = Some(kernel);- key_scales_buffer = Some(scales.buffer());+ key_scales = Some(scales);}Ok(Self {@@ -112,9 +111,8 @@ impl<B: Backend> QKNorm<B> {key_kernel,query_config,key_config,- qkv_array_id,- query_scales_buffer,- key_scales_buffer,+ query_scales,+ key_scales,num_q_heads,num_kv_heads,head_dim,@@ -123,21 +121,19 @@ impl<B: Backend> QKNorm<B> {pub fn encode(&self,- state: &mut ForwardPassState<B>,+ qkv: &mut Allocation<B>,+ batch_dim: usize,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let qkv_array = state.array(self.qkv_array_id);- let batch_dim = qkv_array.shape()[0] as u32;-// Process query normalization if configured- if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =- (&self.query_kernel, &self.query_scales_buffer, &self.query_config)+ if let (Some(query_kernel), Some(query_scales), Some(query_config)) =+ (&self.query_kernel, &self.query_scales, &self.query_config){query_kernel.encode(- None::<&B::DenseBuffer>,- query_scales_buffer.borrow().deref(),- qkv_array.buffer().borrow_mut().deref_mut(),- batch_dim,+ None::<&Allocation<B>>,+ query_scales,+ &mut *qkv,+ batch_dim as u32,self.num_q_heads as u32,self.num_kv_heads as u32,self.head_dim as u32,@@ -151,14 +147,14 @@ impl<B: Backend> QKNorm<B> {}// Process key normalization if configured- if let (Some(key_kernel), Some(key_scales_buffer), Some(key_config)) =- (&self.key_kernel, &self.key_scales_buffer, &self.key_config)+ if let (Some(key_kernel), Some(key_scales), Some(key_config)) =+ (&self.key_kernel, &self.key_scales, &self.key_config){key_kernel.encode(- None::<&B::DenseBuffer>,- key_scales_buffer.borrow().deref(),- qkv_array.buffer().borrow_mut().deref_mut(),- batch_dim,+ None::<&Allocation<B>>,+ key_scales,+ &mut *qkv,+ batch_dim as u32,self.num_q_heads as u32,self.num_kv_heads as u32,self.head_dim as u32,diff --git a/crates/backend-uzu/src/encodable_block/rms_norm.rs b/crates/backend-uzu/src/encodable_block/rms_norm.rsindex 4d07a7242..a45d4fafb 100644--- a/crates/backend-uzu/src/encodable_block/rms_norm.rs+++ b/crates/backend-uzu/src/encodable_block/rms_norm.rs@@ -1,21 +1,15 @@//! RMS Normalization encodable.-use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};-use thiserror::Error;use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, RMSNormKernel},},config::{NormalizationConfig, UpcastMode},- forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};@@ -30,12 +24,11 @@ pub enum RMSNormError<B: Backend> {pub struct RMSNorm<B: Backend> {kernel: <B::Kernels as Kernels>::RMSNormKernel,config: NormalizationConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,- shortcut_array_id: Option<ArrayId>,- scales_buffer: Rc<RefCell<B::DenseBuffer>>,- hadamard_factors_buffer: Option<B::DenseBuffer>,- use_sampling_range: bool,+ scales: Allocation<B>,+ element_count: usize,+ input_data_type: DataType,+ output_data_type: DataType,+ hadamard_factors: Option<Allocation<B>>,}impl<B: Backend> RMSNorm<B> {@@ -43,14 +36,14 @@ impl<B: Backend> RMSNorm<B> {context: &B::Context,intermediate_data_type: DataType,config: NormalizationConfig,- input_array_id: ArrayId,- output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,- hadamard_factors_buffer: Option<B::DenseBuffer>,- shortcut_array_id: Option<ArrayId>,+ hadamard_factors: Option<Allocation<B>>,+ use_shortcut: bool,residual_add: bool,) -> Result<Self, RMSNormError<B>> {- let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;+ let scales_leaf = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;+ let element_count = scales_leaf.shape()[0];+ let scales = scales_leaf.read_allocation().map_err(RMSNormError::ParameterError)?;let accumulation_data_type: DataType = config.accumulation_precision.into();let scale_data_type: DataType = config.scale_precision.into();@@ -66,82 +59,50 @@ impl<B: Backend> RMSNorm<B> {scales_type,output_type,accumulation_data_type,- input_array_id == output_array_id,+ false,config.upcast_mode == UpcastMode::FullLayer,- shortcut_array_id.is_some(),+ use_shortcut,residual_add,- hadamard_factors_buffer.is_some(),+ hadamard_factors.is_some(),).map_err(RMSNormError::BackendError)?;Ok(Self {kernel,config,- input_array_id,- output_array_id,- shortcut_array_id,- scales_buffer: scales.buffer(),- hadamard_factors_buffer,- use_sampling_range: false,+ scales,+ element_count,+ input_data_type: input_type,+ output_data_type: output_type,+ hadamard_factors,})}- /// When enabled, this RMSNorm only runs on `state.sampling_start()..+state.sampling_length()`.- /// This is useful for the final output norm before readout/sampling in prefill.- pub fn with_sampling_range(mut self) -> Self {- self.use_sampling_range = true;- self- }-pub fn encode(&self,- state: &mut ForwardPassState<B>,+ input: &Allocation<B>,+ row_offset: usize,+ row_count: usize,+ shortcut: Option<&mut Allocation<B>>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let input_array = state.array(self.input_array_id);- let output_array = state.array(self.output_array_id);-- let suffix_length = input_array.shape()[0];- let element_count = input_array.shape()[1];-- let input_elem_size = input_array.data_type().size_in_bytes();- let output_elem_size = output_array.data_type().size_in_bytes();-- let (batch_start, batch_len) = if self.use_sampling_range {- (state.sampling_start(), state.sampling_length())- } else {- (0, state.active_row_count())- };-- let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));- if batch_len == 0 {- return Ok(());- }-- let row_size_in_bytes = element_count * input_elem_size;- let input_offset = batch_start * row_size_in_bytes;-- let output_row_size_in_bytes = element_count * output_elem_size;- let output_offset = batch_start * output_row_size_in_bytes;-- let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());- let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());-- let shortcut_rc = self.shortcut_array_id.map(|id| state.array(id).buffer());- let mut shortcut_borrow = shortcut_rc.as_ref().map(|rc| rc.borrow_mut());-+ ) -> Result<Allocation<B>, B::Error> {+ let row_size = self.element_count * self.input_data_type.size_in_bytes();+ let row_offset_bytes = row_offset * row_size;+ let shortcut = shortcut.map(|shortcut| (shortcut, row_offset_bytes));+ let mut output =+ encoder.allocate_scratch(size_for_shape(&[row_count, self.element_count], self.output_data_type))?;self.kernel.encode(- input_buffer_borrow.as_deref().map(|b| (b, input_offset)),- self.scales_buffer.borrow().deref(),- (output_array.buffer().borrow_mut().deref_mut(), output_offset),- shortcut_borrow.as_deref_mut().map(|b| (b, input_offset)),- self.hadamard_factors_buffer.as_ref(),- batch_len as u32,- element_count as u32,+ Some((input, row_offset_bytes)),+ &self.scales,+ &mut output,+ shortcut,+ self.hadamard_factors.as_ref(),+ row_count as u32,+ self.element_count as u32,self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),encoder,);- Ok(())+ Ok(output)}}diff --git a/crates/backend-uzu/src/encodable_block/rope.rs b/crates/backend-uzu/src/encodable_block/rope.rsindex 67978e3af..ed22dd8ca 100644--- a/crates/backend-uzu/src/encodable_block/rope.rs+++ b/crates/backend-uzu/src/encodable_block/rope.rs@@ -1,19 +1,19 @@//! Rope (Rotary Position Embedding) encodable.-use std::ops::{Deref, DerefMut};-use crate::{DataType,+ array::size_for_shape,backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::{Kernels, RopeKernel},},- forward_pass::state::{ArrayId, ForwardPassState, RopeType},+ forward_pass::state::RopeType,};pub struct Rope<B: Backend> {kernel: <B::Kernels as Kernels>::RopeKernel,rope_type: RopeType,+ data_type: DataType,}impl<B: Backend> Rope<B> {@@ -25,51 +25,54 @@ impl<B: Backend> Rope<B> {Ok(Self {kernel: <B::Kernels as Kernels>::RopeKernel::new(context, data_type)?,rope_type,+ data_type,})}pub fn encode(&self,- state: &mut ForwardPassState<B>,+ qkv: &Allocation<B>,+ token_positions: &Allocation<B>,+ cosines: &Allocation<B>,+ sines: &Allocation<B>,+ suffix_length: usize,+ num_heads: usize,+ num_groups: usize,+ head_dim: usize,+ rope_max_seq_len: usize,+ rope_dim: usize,use_rope: bool,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let token_positions = state.array(ArrayId::TokenPositions);- let qkv = state.array(ArrayId::QKV);- let cosines = state.array(ArrayId::RopeCosines(self.rope_type));- let sines = state.array(ArrayId::RopeSines(self.rope_type));- let rotated_queries = state.array(ArrayId::RotatedQueries);- let rotated_keys = state.array(ArrayId::RotatedKeys);-- let suffix_length = qkv.shape()[0];-- let rope_max_seq_len = cosines.shape()[0];- let rope_dim = if use_rope {- cosines.shape()[1]+ ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {+ let applied_rope_dim = if use_rope {+ assert!(rope_dim > 0, "enabled RoPE dimension must be greater than zero");+ rope_dim} else {0};-- let num_heads = rotated_queries.shape()[0];- let head_dim = rotated_queries.shape()[2];-- let num_groups = rotated_keys.shape()[0];-+ let mut rotated_queries =+ encoder.allocate_scratch(size_for_shape(&[num_heads, suffix_length, head_dim], self.data_type))?;+ let mut rotated_keys =+ encoder.allocate_scratch(size_for_shape(&[num_groups, suffix_length, head_dim], self.data_type))?;self.kernel.encode(- qkv.buffer().borrow().deref(),- cosines.buffer().borrow().deref(),- sines.buffer().borrow().deref(),- (token_positions.buffer().borrow().deref(), token_positions.offset()),- rotated_queries.buffer().borrow_mut().deref_mut(),- rotated_keys.buffer().borrow_mut().deref_mut(),+ qkv,+ cosines,+ sines,+ token_positions,+ &mut rotated_queries,+ &mut rotated_keys,head_dim as u32,- rope_dim as u32,+ applied_rope_dim as u32,num_heads as u32,num_groups as u32,suffix_length as u32,rope_max_seq_len as u32,encoder,);- Ok(())+ Ok((rotated_queries, rotated_keys))+ }++ pub fn rope_type(&self) -> RopeType {+ self.rope_type}}diff --git a/crates/backend-uzu/src/encodable_block/sampling.rs b/crates/backend-uzu/src/encodable_block/sampling.rsindex 26fdd9173..47c2c30ca 100644--- a/crates/backend-uzu/src/encodable_block/sampling.rs+++ b/crates/backend-uzu/src/encodable_block/sampling.rs@@ -1,28 +1,79 @@//! Sampling kernel encodable.-use std::ops::{Deref, DerefMut};+use ndarray::ArrayView2;use crate::{DataType,+ array::{Array, ArrayContextExt},backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder,kernel::sampling::{SamplingError, SamplingKernel},},- forward_pass::state::{ArrayId, ForwardPassState},+ session::parameter::SamplingMethod,};pub struct Sampling<B: Backend> {kernel: SamplingKernel<B>,}+pub struct SamplingArguments<'a, B: Backend> {+ pub logits: &'a mut Allocation<B>,+ pub seeds: &'a Allocation<B>,+ pub bitmask: Option<&'a Allocation<B>>,+ pub output: &'a mut Allocation<B>,+ pub sampling_method: SamplingMethod,+ pub batch_size: usize,+ pub vocab_size: usize,+}++pub(crate) struct SamplingInputs<B: Backend> {+ pub seeds: Array<B>,+ pub bitmask: Option<Array<B>>,+}++impl<B: Backend> SamplingInputs<B> {+ pub(crate) fn bitmask_array_from_slice(+ context: &B::Context,+ row_count: usize,+ token_bitmask: &[u32],+ bitmask_row_len: usize,+ ) -> Array<B> {+ let mut array = context.create_array_zeros(&[row_count, bitmask_row_len], DataType::U32);+ let source =+ ArrayView2::from_shape((row_count, bitmask_row_len), token_bitmask).expect("Invalid token bitmask shape");+ array.copy_from_view(source);+ array+ }++ pub(crate) fn from_slices(+ context: &B::Context,+ token_seeds: &[u64],+ token_bitmask: Option<&[u32]>,+ bitmask_row_len: Option<usize>,+ ) -> Self {+ let seeds = context.create_array_from(&[token_seeds.len()], token_seeds);++ let bitmask = match (token_bitmask, bitmask_row_len) {+ (Some(token_bitmask), Some(bitmask_row_len)) => {+ Some(Self::bitmask_array_from_slice(context, token_seeds.len(), token_bitmask, bitmask_row_len))+ },+ (None, None) => None,+ _ => panic!("bitmask data and row length must either both exist or both be absent"),+ };++ Self {+ seeds,+ bitmask,+ }+ }+}+impl<B: Backend> Sampling<B> {pub fn new(context: &B::Context,data_type: DataType,- max_batch_size: usize,- max_vocab_size: usize,) -> Result<Self, SamplingError<B>> {- let kernel = SamplingKernel::new(context, data_type, max_batch_size, max_vocab_size)?;+ let kernel = SamplingKernel::new(context, data_type)?;Ok(Self {kernel,})@@ -30,43 +81,19 @@ impl<B: Backend> Sampling<B> {pub fn encode(&self,- state: &mut ForwardPassState<B>,+ args: SamplingArguments<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let batch_dim = state.sampling_length();- let sampling_start = state.sampling_start();- let sampling_method = state.sampling_method().unwrap();-- let logits = state.array(ArrayId::Logits);- let seeds = state.array(ArrayId::TokenSeeds);- let output = state.sampling_output().unwrap();-- let vocab_size = logits.shape()[1];- let seeds_offset = seeds.offset() + sampling_start * std::mem::size_of::<u64>();-- let (bitmask_buffer, bitmask_offset) = state.token_bitmask().map_or((None, 0usize), |bitmask| {- let bitmask_row_len = bitmask.shape()[1];- let bitmask_offset = bitmask.offset() + sampling_start * bitmask_row_len * std::mem::size_of::<u32>();- (Some(bitmask.buffer()), bitmask_offset)- });- let bitmask_borrow = bitmask_buffer.as_ref().map(|b| b.borrow());-- self.kernel- .encode(- logits.buffer().borrow_mut().deref_mut(),- seeds.buffer().borrow().deref(),- seeds_offset,- bitmask_borrow.as_deref(),- bitmask_offset,- output.buffer().borrow_mut().deref_mut(),- sampling_method,- batch_dim,- vocab_size,- encoder,- )- .expect("Sampling encoding failed");-- Ok(())+ ) -> Result<(), SamplingError<B>> {+ self.kernel.encode(+ args.logits,+ args.seeds,+ args.bitmask,+ args.output,+ args.sampling_method,+ args.batch_size,+ args.vocab_size,+ encoder,+ )}}diff --git a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rsindex 7d5318999..e7eca1e62 100644--- a/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/backend-uzu/src/encodable_block/short_conv_mixer.rs@@ -1,20 +1,31 @@-use std::ops::{Deref, DerefMut};+use thiserror::Error;use crate::{DataType,- array::Array,+ array::size_for_shape,backends::common::{- Backend, Context, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},},config::{DecoderLayerType, ShortConvConfig},- encodable_block::linear::Linear,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::{ParameterTree, resolve_subtree},+ encodable_block::linear::{Linear, LinearBlockError},+ forward_pass::short_conv_layer::ShortConvLayer,+ parameters::{ParameterLoaderError, ParameterTree, try_resolve_subtree},};+#[derive(Debug, Error)]+pub enum ShortConvMixerError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Unsupported configuration: {0}")]+ UnsupportedConfiguration(String),+ #[error("Linear error: {0}")]+ LinearError(#[from] Box<LinearBlockError<B>>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+}+pub struct ShortConvMixer<B: Backend> {- layer_index: usize,config: ShortConvConfig,model_dim: usize,in_projection: Box<dyn Linear<B>>,@@ -23,8 +34,17 @@ pub struct ShortConvMixer<B: Backend> {short_conv_prefill: <B::Kernels as Kernels>::ShortConvPrefillKernel,short_conv_decode: <B::Kernels as Kernels>::ShortConvDecodeKernel,short_conv_trie: <B::Kernels as Kernels>::ShortConvTrieKernel,- conv_weight: Array<B>,- conv_bias: Option<Array<B>>,+ conv_weight: Allocation<B>,+ conv_bias: Option<Allocation<B>>,+ data_type: DataType,+}++pub(crate) struct ShortConvArguments<'a, B: Backend> {+ pub active_row_count: usize,+ pub sampling_start: usize,+ pub sampling_length: usize,+ pub token_parents: &'a Allocation<B>,+ pub layer: &'a mut ShortConvLayer<B>,}impl<B: Backend> ShortConvMixer<B> {@@ -35,60 +55,61 @@ impl<B: Backend> ShortConvMixer<B> {layer_index: usize,model_dim: usize,decoder_layer_loader: &ParameterTree<B::Context>,- ) -> (Self, Option<B::DenseBuffer>) {+ ) -> Result<(Self, Option<Allocation<B>>), ShortConvMixerError<B>> {if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {- panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);+ return Err(ShortConvMixerError::UnsupportedConfiguration(format!(+ "layer {layer_index} marked as non-ShortConv but ShortConv config provided"+ )));+ }+ if short_conv_config.kernel_size < 2 {+ return Err(ShortConvMixerError::UnsupportedConfiguration(format!(+ "kernel_size must be >= 2, got {}",+ short_conv_config.kernel_size+ )));}- let mixer_tree = resolve_subtree(decoder_layer_loader, &["mixer"]);- let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);+ let mixer_tree = try_resolve_subtree(decoder_layer_loader, &["mixer"])?;+ let conv_tree = try_resolve_subtree(&mixer_tree, &["conv"])?;let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();let (in_projection, in_proj_input_hadamard_factors) = <dyn Linear<B>>::new_extracting_input_hadamard(&short_conv_config.in_projection_config,- false,model_dim,[model_dim * 3],context,- &resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),- ArrayId::Main,- ArrayId::SsmInProj,+ &try_resolve_subtree(&mixer_tree, &["in_projection", "in_proj"])?,)- .expect("Failed to create in-projection kernel");+ .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;let out_projection = <dyn Linear<B>>::new(&short_conv_config.out_projection_config,- false,model_dim,[model_dim],context,- &resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),- ArrayId::AttentionOutput,- ArrayId::Main,+ &try_resolve_subtree(&mixer_tree, &["out_projection", "out_proj"])?,)- .expect("Failed to create out-projection kernel");+ .map_err(|err| ShortConvMixerError::LinearError(Box::new(err)))?;- let conv_weight = conv_tree.leaf_array("weights").unwrap().clone();+ let conv_weight = conv_tree.leaf("weights")?.read_allocation()?;let conv_bias = if short_conv_config.conv_config.has_biases {- Some(conv_tree.leaf_array("biases").unwrap().clone())+ Some(conv_tree.leaf("biases")?.read_allocation()?)} else {None};let has_bias = short_conv_config.conv_config.has_biases;let short_conv_pack = <B::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)- .expect("Failed to create short conv pack kernel");+ .map_err(ShortConvMixerError::BackendError)?;let short_conv_prefill = <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)- .expect("Failed to create short conv prefill kernel");+ .map_err(ShortConvMixerError::BackendError)?;let short_conv_decode = <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias, true)- .expect("Failed to create short conv decode kernel");+ .map_err(ShortConvMixerError::BackendError)?;let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)- .expect("Failed to create short conv trie kernel");+ .map_err(ShortConvMixerError::BackendError)?;- (+ Ok((Self {- layer_index,config: short_conv_config,model_dim,in_projection,@@ -99,169 +120,130 @@ impl<B: Backend> ShortConvMixer<B> {short_conv_trie,conv_weight,conv_bias,+ data_type,},in_proj_input_hadamard_factors,- )- }-- fn clear_suffix_state_valid_range(- &self,- state: &ForwardPassState<B>,- ) {- let Some(cache_layers) = state.cache_layers() else {- return;- };- let cache = cache_layers.borrow();- let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");- layer.clear_suffix_state_valid_range();- }-- fn set_suffix_state_valid_range(- &self,- state: &ForwardPassState<B>,- start: usize,- len: usize,- ) {- let Some(cache_layers) = state.cache_layers() else {- return;- };- let cache = cache_layers.borrow();- let layer = cache.data[self.layer_index].as_short_conv().expect("Expected ShortConv layer");- layer.set_suffix_state_valid_range(start, len);+ ))}fn run_conv(&self,- state: &mut ForwardPassState<B>,+ layer: &mut ShortConvLayer<B>,+ token_parents: &Allocation<B>,+ sampling_start: usize,+ sampling_length: usize,+ in_proj: &Allocation<B>,encoder: &mut Encoder<B>,active_row_count: usize,- ) {- self.clear_suffix_state_valid_range(state);+ ) -> Result<Allocation<B>, B::Error> {+ layer.clear_suffix_state_valid_range();+ let mut out = encoder.allocate_scratch(size_for_shape(&[active_row_count, self.model_dim], self.data_type))?;if active_row_count == 1 {- self.run_decode_conv(state, encoder, 1);- return;+ self.run_decode_conv(layer, in_proj, &mut out, encoder, 1)?;+ return Ok(out);}- let sampling_len = state.sampling_length();- if sampling_len == 0 {- self.run_prefill_conv(state, encoder, active_row_count);- return;+ if sampling_length == 0 {+ self.run_prefill_conv(layer, in_proj, &mut out, encoder, active_row_count)?;+ return Ok(out);}- let sampling_start = state.sampling_start();- let trie_len = sampling_len;+ let trie_len = sampling_length;if trie_len <= 1 {- self.run_prefill_conv(state, encoder, active_row_count);- return;+ self.run_prefill_conv(layer, in_proj, &mut out, encoder, active_row_count)?;+ return Ok(out);}if sampling_start > 0 {if sampling_start == 1 {- self.run_decode_conv(state, encoder, 1);+ self.run_decode_conv(layer, in_proj, &mut out, encoder, 1)?;} else {- self.run_prefill_conv(state, encoder, sampling_start);+ self.run_prefill_conv(layer, in_proj, &mut out, encoder, sampling_start)?;}}- self.run_trie_conv(state, encoder, sampling_start, trie_len);- self.set_suffix_state_valid_range(state, sampling_start, trie_len);+ self.run_trie_conv(layer, token_parents, in_proj, &mut out, encoder, sampling_start, trie_len)?;+ layer.set_suffix_state_valid_range(sampling_start, trie_len);+ Ok(out)}fn run_prefill_conv(&self,- state: &mut ForwardPassState<B>,+ layer: &mut ShortConvLayer<B>,+ in_proj: &Allocation<B>,+ out: &mut Allocation<B>,encoder: &mut Encoder<B>,suffix_length: usize,- ) {+ ) -> Result<(), B::Error> {if self.model_dim == 0 || suffix_length == 0 {- return;+ return Ok(());}- let in_proj = state.array(ArrayId::SsmInProj);- let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));- let out = state.array(ArrayId::AttentionOutput);-- let bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());-let kernel_size = self.config.kernel_size;- let state_stride = kernel_size.saturating_sub(1);+ let state_stride = kernel_size - 1;- // Allocate temporary padded buffer- let data_type: DataType = self.config.in_projection_config.activation_precision().into();- let element_size = data_type.size_in_bytes();let padded_rows = state_stride + suffix_length;- let padded_size = padded_rows * self.model_dim * element_size;- let mut padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");+ let mut padded = encoder.allocate_scratch(size_for_shape(&[padded_rows, self.model_dim], self.data_type))?;self.short_conv_pack.encode(- conv_state.buffer().borrow().deref(),- in_proj.buffer().borrow().deref(),- &mut padded_buf,+ (&layer.conv_state, 0),+ in_proj,+ &mut padded,state_stride as u32,suffix_length as u32,self.model_dim as u32 * 3,self.model_dim as u32,encoder,);-self.short_conv_prefill.encode(- &padded_buf,- in_proj.buffer().borrow().deref(),- self.conv_weight.buffer().borrow().deref(),- bias_buf_borrow.as_deref(),- out.buffer().borrow_mut().deref_mut(),- conv_state.buffer().borrow_mut().deref_mut(),+ &padded,+ in_proj,+ &self.conv_weight,+ self.conv_bias.as_ref(),+ out,+ &mut layer.conv_state,suffix_length as u32,kernel_size as u32,self.model_dim as u32 * 3,state_stride as u32,self.model_dim as u32,encoder,- )+ );+ Ok(())}fn run_trie_conv(&self,- state: &mut ForwardPassState<B>,+ layer: &mut ShortConvLayer<B>,+ token_parents: &Allocation<B>,+ in_proj: &Allocation<B>,+ out: &mut Allocation<B>,encoder: &mut Encoder<B>,sampling_start: usize,trie_len: usize,- ) {+ ) -> Result<(), B::Error> {if self.model_dim == 0 || trie_len == 0 {- return;+ return Ok(());}- let in_proj = state.array(ArrayId::SsmInProj);- let parents = state.array(ArrayId::TokenParents);- let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));- let suffix_state = state.array(ArrayId::ShortConvSuffixState(self.layer_index));- let out = state.array(ArrayId::AttentionOutput);-- let data_type: DataType = self.config.in_projection_config.activation_precision().into();- let elem_bytes = data_type.size_in_bytes();+ let elem_bytes = DataType::from(self.config.in_projection_config.activation_precision()).size_in_bytes();let kernel_size = self.config.kernel_size;- let state_stride = kernel_size.saturating_sub(1);+ let state_stride = kernel_size - 1;let in_proj_stride = self.model_dim * 3;- let in_proj_offset = in_proj.offset() + sampling_start * in_proj_stride * elem_bytes;- let out_offset = out.offset() + sampling_start * self.model_dim * elem_bytes;- let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;- let base_state_offset = conv_state.offset();- let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();- let trie_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());- let trie_bias_buf_borrow = trie_bias_buf_rc.as_ref().map(|rc| rc.borrow());-+ let in_proj_view = (in_proj, sampling_start * in_proj_stride * elem_bytes);+ let parents_view = (token_parents, sampling_start * std::mem::size_of::<i32>());+ let out_view = (&mut *out, sampling_start * self.model_dim * elem_bytes);self.short_conv_trie.encode(- (in_proj.buffer().borrow().deref(), in_proj_offset),- self.conv_weight.buffer().borrow().deref(),- trie_bias_buf_borrow.as_deref(),- (conv_state.buffer().borrow().deref(), base_state_offset),- (parents.buffer().borrow().deref(), parents_offset),- (out.buffer().borrow_mut().deref_mut(), out_offset),- (suffix_state.buffer().borrow_mut().deref_mut(), suffix_state_offset),+ in_proj_view,+ &self.conv_weight,+ self.conv_bias.as_ref(),+ (&layer.conv_state, 0),+ parents_view,+ out_view,+ (&mut layer.suffix_state, sampling_start * self.model_dim * state_stride * elem_bytes),trie_len as u32,kernel_size as u32,in_proj_stride as u32,@@ -269,60 +251,60 @@ impl<B: Backend> ShortConvMixer<B> {self.model_dim as u32,encoder,);+ Ok(())}fn run_decode_conv(&self,- state: &mut ForwardPassState<B>,+ layer: &mut ShortConvLayer<B>,+ in_proj: &Allocation<B>,+ out: &mut Allocation<B>,encoder: &mut Encoder<B>,suffix_length: usize,- ) {+ ) -> Result<(), B::Error> {if self.model_dim == 0 || suffix_length == 0 {- return;+ return Ok(());}- let in_proj = state.array(ArrayId::SsmInProj);- let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));- let out = state.array(ArrayId::AttentionOutput);-- let decode_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());- let decode_bias_buf_borrow = decode_bias_buf_rc.as_ref().map(|rc| rc.borrow());-let kernel_size = self.config.kernel_size;- let state_stride = kernel_size.saturating_sub(1);+ let state_stride = kernel_size - 1;self.short_conv_decode.encode(- in_proj.buffer().borrow().deref(),- self.conv_weight.buffer().borrow().deref(),- decode_bias_buf_borrow.as_deref(),- None::<&B::DenseBuffer>,- out.buffer().borrow_mut().deref_mut(),- conv_state.buffer().borrow_mut().deref_mut(),+ in_proj,+ &self.conv_weight,+ self.conv_bias.as_ref(),+ None::<&Allocation<B>>,+ out,+ &mut layer.conv_state,suffix_length as u32,kernel_size as u32,self.model_dim as u32 * 3,state_stride as u32,self.model_dim as u32,encoder,- )+ );+ Ok(())}pub fn encode(&self,- state: &mut ForwardPassState<B>,+ args: ShortConvArguments<B>,+ input: Allocation<B>,encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let active_row_count = state.active_row_count();- if active_row_count == 0 {- return Ok(());- }-- self.in_projection.encode(state, encoder)?;-- self.run_conv(state, encoder, active_row_count);-- self.out_projection.encode(state, encoder)?;-- Ok(())+ ) -> Result<Allocation<B>, B::Error> {+ let ShortConvArguments {+ active_row_count,+ sampling_start,+ sampling_length,+ token_parents,+ layer,+ } = args;+ assert!(active_row_count > 0, "ShortConv mixer requires at least one active row");++ let in_proj = self.in_projection.encode(input, active_row_count, encoder)?;+ let conv_output =+ self.run_conv(layer, token_parents, sampling_start, sampling_length, &in_proj, encoder, active_row_count)?;++ self.out_projection.encode(conv_output, active_row_count, encoder)}}diff --git a/crates/backend-uzu/src/encodable_block/tensor_add_swap.rs b/crates/backend-uzu/src/encodable_block/tensor_add_swap.rsdeleted file mode 100644index 2320c2c4a..000000000--- a/crates/backend-uzu/src/encodable_block/tensor_add_swap.rs+++ /dev/null@@ -1,54 +0,0 @@-//! Tensor add-swap encodable.--use std::ops::DerefMut;--use crate::{- DataType,- backends::common::{- Backend, Encoder,- kernel::{Kernels, TensorAddSwapKernel},- },- forward_pass::state::{ArrayId, ForwardPassState},-};--pub struct TensorAddSwap<B: Backend> {- kernel: <B::Kernels as Kernels>::TensorAddSwapKernel,- skip_array_id: ArrayId,- main_array_id: ArrayId,-}--impl<B: Backend> TensorAddSwap<B> {- pub fn new(- context: &B::Context,- data_type: DataType,- skip_array_id: ArrayId,- main_array_id: ArrayId,- ) -> Result<Self, B::Error> {- let kernel = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)?;-- Ok(Self {- kernel,- skip_array_id,- main_array_id,- })- }-- pub fn encode(- &self,- state: &mut ForwardPassState<B>,- encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let skip_array = state.array(self.skip_array_id);- let main_array = state.array(self.main_array_id);-- let length = skip_array.num_elements();-- self.kernel.encode(- skip_array.buffer().borrow_mut().deref_mut(),- main_array.buffer().borrow_mut().deref_mut(),- length as u32,- encoder,- );- Ok(())- }-}diff --git a/crates/backend-uzu/src/encodable_block/tensor_copy.rs b/crates/backend-uzu/src/encodable_block/tensor_copy.rsdeleted file mode 100644index 69e6c9117..000000000--- a/crates/backend-uzu/src/encodable_block/tensor_copy.rs+++ /dev/null@@ -1,54 +0,0 @@-//! Tensor copy encodable.--use std::ops::{Deref, DerefMut};--use crate::{- DataType,- backends::common::{- Backend, Encoder,- kernel::{Kernels, TensorCopyKernel},- },- forward_pass::state::{ArrayId, ForwardPassState},-};--pub struct TensorCopy<B: Backend> {- kernel: <B::Kernels as Kernels>::TensorCopyKernel,- source_array: ArrayId,- destination_array: ArrayId,-}--impl<B: Backend> TensorCopy<B> {- pub fn new(- context: &B::Context,- data_type: DataType,- source_array: ArrayId,- destination_array: ArrayId,- ) -> Result<Self, B::Error> {- let kernel = <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type)?;-- Ok(Self {- kernel,- source_array,- destination_array,- })- }-- pub fn encode(- &self,- state: &mut ForwardPassState<B>,- encoder: &mut Encoder<B>,- ) -> Result<(), B::Error> {- let source_array = state.array(self.source_array);- let destination_array = state.array(self.destination_array);-- let length = source_array.num_elements();-- self.kernel.encode(- source_array.buffer().borrow().deref(),- destination_array.buffer().borrow_mut().deref_mut(),- length as u32,- encoder,- );- Ok(())- }-}diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rsindex f99d46aa8..26b06f4ab 100644--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs@@ -1,8 +1,8 @@use std::cell::Cell;use crate::{- array::ArrayContextExt,- backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},+ array::size_for_shape,+ backends::common::{AllocationType, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},config::DecoderLayerType,forward_pass::{delta_net_layer::DeltaNetLayer,@@ -13,7 +13,6 @@ use crate::{},};-#[derive(Debug)]pub enum CacheLayer<B: Backend> {Transformer(KVCacheLayer<B>),StateSpace(SSMLayer<B>),@@ -21,7 +20,6 @@ pub enum CacheLayer<B: Backend> {DeltaNet(DeltaNetLayer<B>),}-#[derive(Clone)]pub enum CacheLayerSlice<B: Backend> {Transformer(KVSlice<B>),StateSpace,@@ -29,15 +27,6 @@ pub enum CacheLayerSlice<B: Backend> {DeltaNet,}-const ARRAY_TRANSFORMER_KEYS_LABEL: &str = "cache_layers_transformer_keys";-const ARRAY_TRANSFORMER_VALUES_LABEL: &str = "cache_layers_transformer_values";-const ARRAY_STATE_SPACE_CONV_STATE_LABEL: &str = "cache_layers_state_space_conv_state";-const ARRAY_STATE_SPACE_SSM_STATE_LABEL: &str = "cache_layers_state_space_ssm_state";-const ARRAY_SHORT_CONV_CONV_STATE_LABEL: &str = "cache_layers_short_conv_conv_state";-const ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL: &str = "cache_layers_short_conv_suffix_state";-const ARRAY_DELTA_NET_CONV_STATE_LABEL: &str = "cache_layers_delta_net_conv_state";-const ARRAY_DELTA_NET_SSM_STATE_LABEL: &str = "cache_layers_delta_net_ssm_state";-impl<B: Backend> CacheLayer<B> {pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {match self {@@ -102,7 +91,6 @@ pub struct CacheLayers<B: Backend> {pub data: Box<[CacheLayer<B>]>,}-#[derive(Clone)]pub struct CacheLayersSlice<B: Backend> {pub layers: Vec<CacheLayerSlice<B>>,}@@ -118,7 +106,7 @@ impl<B: Backend> CacheLayers<B> {let kv_shapes: Vec<[usize; 3]> =model_shape.kv_cache_layer_shapes(max_prefix_length, max_suffix_length).collect();- let data: Box<[CacheLayer<B>]> = model_shape+ let mut data: Box<[CacheLayer<B>]> = model_shape.layer_types().iter().enumerate()@@ -139,19 +127,18 @@ impl<B: Backend> CacheLayers<B> {prefix_len: 0,}};+ let kv_bytes = size_for_shape(&shape, model_shape.kv_cache_data_type());CacheLayer::Transformer(KVCacheLayer {state: state.clone(),- keys: context.create_array_zeros(- &shape,- model_shape.kv_cache_data_type(),- &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),- ),- values: context.create_array_zeros(- &shape,- model_shape.kv_cache_data_type(),- &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),- ),+ keys: context+ .create_allocation(kv_bytes, AllocationType::Global)+ .expect("Failed to create kv keys allocation"),+ values: context+ .create_allocation(kv_bytes, AllocationType::Global)+ .expect("Failed to create kv values allocation"),+ shape,+ data_type: model_shape.kv_cache_data_type(),})},DecoderLayerType::StateSpace {@@ -165,39 +152,43 @@ impl<B: Backend> CacheLayers<B> {let conv_shape = [*conv_dim, kernel_size.saturating_sub(1)];let ssm_shape = [*num_heads, *head_dim, *state_dim];let dtype = model_shape.activation_data_type();+ let conv_bytes = size_for_shape(&conv_shape, dtype);+ let ssm_bytes = size_for_shape(&ssm_shape, dtype);CacheLayer::StateSpace(SSMLayer {- conv_state: context.create_array_zeros(- &conv_shape,- dtype,- &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),- ),- ssm_state: context.create_array_zeros(- &ssm_shape,- dtype,- &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),- ),+ conv_state: (conv_bytes > 0).then(|| {+ context+ .create_allocation(conv_bytes, AllocationType::Global)+ .expect("Failed to create ssm conv allocation")+ }),+ conv_shape,+ ssm_state: context+ .create_allocation(ssm_bytes, AllocationType::Global)+ .expect("Failed to create ssm state allocation"),+ ssm_shape,+ data_type: dtype,})},DecoderLayerType::ShortConv {kernel_size,} => {- let conv_shape = [model_shape.model_dim(), kernel_size.saturating_sub(1)];- let suffix_state_shape =- [max_suffix_length, model_shape.model_dim(), kernel_size.saturating_sub(1)];+ assert!(*kernel_size >= 2, "ShortConv kernel_size must be >= 2, got {}", kernel_size);+ let conv_shape = [model_shape.model_dim(), kernel_size - 1];+ let suffix_state_shape = [max_suffix_length, model_shape.model_dim(), kernel_size - 1];let dtype = model_shape.activation_data_type();+ let conv_bytes = size_for_shape(&conv_shape, dtype);+ let suffix_bytes = size_for_shape(&suffix_state_shape, dtype);CacheLayer::ShortConv(ShortConvLayer {- conv_state: context.create_array_zeros(- &conv_shape,- dtype,- &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),- ),- suffix_state: context.create_array_zeros(- &suffix_state_shape,- dtype,- &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),- ),+ conv_state: context+ .create_allocation(conv_bytes, AllocationType::Global)+ .expect("Failed to create short conv allocation"),+ conv_shape,+ suffix_state: context+ .create_allocation(suffix_bytes, AllocationType::Global)+ .expect("Failed to create short conv suffix allocation"),+ suffix_shape: suffix_state_shape,+ data_type: dtype,suffix_state_valid_start: Cell::new(0),suffix_state_valid_len: Cell::new(0),})@@ -213,23 +204,49 @@ impl<B: Backend> CacheLayers<B> {let conv_shape = [*conv_dim, kernel_size.saturating_sub(1)];let ssm_shape = [*num_heads, *value_head_dim, *head_dim];let dtype = model_shape.activation_data_type();+ let conv_bytes = size_for_shape(&conv_shape, dtype);+ let ssm_bytes = size_for_shape(&ssm_shape, dtype);CacheLayer::DeltaNet(DeltaNetLayer {- conv_state: context.create_array_zeros(- &conv_shape,- dtype,- &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),- ),- ssm_state: context.create_array_zeros(- &ssm_shape,- dtype,- &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),- ),+ conv_state: context+ .create_allocation(conv_bytes, AllocationType::Global)+ .expect("Failed to create delta net conv allocation"),+ conv_shape,+ ssm_state: context+ .create_allocation(ssm_bytes, AllocationType::Global)+ .expect("Failed to create delta net ssm allocation"),+ ssm_shape,+ data_type: dtype,})},}).collect();+ let mut encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache initialization encoder");+ for layer in data.iter_mut() {+ match layer {+ CacheLayer::Transformer(layer) => {+ encoder.encode_fill(&mut layer.keys, 0);+ encoder.encode_fill(&mut layer.values, 0);+ },+ CacheLayer::StateSpace(layer) => {+ if let Some(conv_state) = layer.conv_state.as_mut() {+ encoder.encode_fill(conv_state, 0);+ }+ encoder.encode_fill(&mut layer.ssm_state, 0);+ },+ CacheLayer::ShortConv(layer) => {+ encoder.encode_fill(&mut layer.conv_state, 0);+ encoder.encode_fill(&mut layer.suffix_state, 0);+ },+ CacheLayer::DeltaNet(layer) => {+ encoder.encode_fill(&mut layer.conv_state, 0);+ encoder.encode_fill(&mut layer.ssm_state, 0);+ },+ }+ }+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to initialize cache allocations");+Self {max_suffix_length,max_prefix_length,@@ -237,7 +254,11 @@ impl<B: Backend> CacheLayers<B> {}}- pub fn clear(&mut self) {+ pub fn clear(+ &mut self,+ context: &B::Context,+ ) {+ let mut encoder: Option<Encoder<B>> = None;for layer in self.data.iter_mut() {match layer {CacheLayer::Transformer(layer) => match &mut layer.state {@@ -255,11 +276,32 @@ impl<B: Backend> CacheLayers<B> {*ring_length = 0;},},- CacheLayer::StateSpace(layer) => layer.zero(),- CacheLayer::ShortConv(layer) => layer.zero(),- CacheLayer::DeltaNet(layer) => layer.zero(),+ CacheLayer::StateSpace(layer) => {+ let encoder = encoder+ .get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));+ if let Some(conv_state) = layer.conv_state.as_mut() {+ encoder.encode_fill(conv_state, 0);+ }+ encoder.encode_fill(&mut layer.ssm_state, 0);+ },+ CacheLayer::ShortConv(layer) => {+ let encoder = encoder+ .get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));+ encoder.encode_fill(&mut layer.conv_state, 0);+ encoder.encode_fill(&mut layer.suffix_state, 0);+ layer.clear_suffix_state_valid_range();+ },+ CacheLayer::DeltaNet(layer) => {+ let encoder = encoder+ .get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));+ encoder.encode_fill(&mut layer.conv_state, 0);+ encoder.encode_fill(&mut layer.ssm_state, 0);+ },}}+ if let Some(encoder) = encoder {+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to clear cache layers");+ }}pub fn max_suffix_length(&self) -> usize {@@ -282,7 +324,7 @@ impl<B: Backend> CacheLayers<B> {if let Some(layer) = layer.as_transformer_mut() {layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);} else if let Some(layer) = layer.as_short_conv_mut() {- layer.commit_from_suffix_state_if_valid(short_conv_commit_index);+ layer.commit_from_suffix_state_if_valid(short_conv_commit_index, encoder);}}}@@ -340,21 +382,94 @@ impl<B: Backend> CacheLayers<B> {}}+ pub fn copy_from(+ &mut self,+ source: &Self,+ context: &B::Context,+ ) {+ self.copy_metadata_and_windowed_data_from(source, context);++ let mut encoder = Encoder::new(context).expect("Failed to create cache layer copy encoder");+ self.encode_copy_data_from(source, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to copy cache layers");+ }++ fn copy_metadata_and_windowed_data_from(+ &mut self,+ source: &Self,+ context: &B::Context,+ ) {+ assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");++ for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {+ match (source_layer, destination_layer) {+ (CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {+ let copy_rows = source.prefix_segment_length();+ if copy_rows > 0 && matches!(source.state, KVCacheLayerState::Windowed { .. }) {+ let slice = source.slice(context, 0..copy_rows).expect("Failed to slice KV cache layer");+ destination.apply_slice(&slice, None);+ }+ destination.state = source.state.clone();+ },+ (CacheLayer::StateSpace(_), CacheLayer::StateSpace(_)) => {},+ (CacheLayer::ShortConv(_), CacheLayer::ShortConv(destination)) => {+ destination.clear_suffix_state_valid_range();+ },+ (CacheLayer::DeltaNet(_), CacheLayer::DeltaNet(_)) => {},+ _ => panic!("cache layer type mismatch while copying cache layers"),+ }+ }+ }++ fn encode_copy_data_from(+ &mut self,+ source: &Self,+ encoder: &mut Encoder<B>,+ ) {+ assert_eq!(source.data.len(), self.data.len(), "cache layer count mismatch");++ for (source_layer, destination_layer) in source.data.iter().zip(self.data.iter_mut()) {+ match (source_layer, destination_layer) {+ (CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {+ if matches!(source.state, KVCacheLayerState::Full { .. }) {+ source.encode_copy_prefix_rows_to(destination, source.prefix_segment_length(), encoder);+ }+ },+ (CacheLayer::StateSpace(source), CacheLayer::StateSpace(destination)) => {+ match (source.conv_state.as_ref(), destination.conv_state.as_mut()) {+ (Some(source_conv_state), Some(destination_conv_state)) => {+ encoder.encode_copy(source_conv_state, .., destination_conv_state, ..);+ },+ (None, None) => {},+ _ => panic!("state-space conv_state presence mismatch while copying cache layers"),+ }+ encoder.encode_copy(&source.ssm_state, .., &mut destination.ssm_state, ..);+ },+ (CacheLayer::ShortConv(source), CacheLayer::ShortConv(destination)) => {+ encoder.encode_copy(&source.conv_state, .., &mut destination.conv_state, ..);+ },+ (CacheLayer::DeltaNet(source), CacheLayer::DeltaNet(destination)) => {+ encoder.encode_copy(&source.conv_state, .., &mut destination.conv_state, ..);+ encoder.encode_copy(&source.ssm_state, .., &mut destination.ssm_state, ..);+ },+ _ => panic!("cache layer type mismatch while copying cache layers"),+ }+ }+ }+pub fn clone(&self,context: &B::Context,) -> Self {let mut max_prefix_capacity_across_layers = 0usize;- let data: Box<[CacheLayer<B>]> = self+ let mut data: Box<[CacheLayer<B>]> = self.data.iter()- .enumerate()- .map(|(layer_index, layer)| match layer {+ .map(|layer| match layer {CacheLayer::Transformer(layer) => {- let shape = layer.keys.shape().to_vec();- let num_groups = shape[1];- let head_dim = shape[2];- let dtype = layer.keys.data_type();+ let shape = layer.shape;+ let [_, num_groups, head_dim] = shape;+ let dtype = layer.data_type;let copy_rows = layer.prefix_segment_length();let new_total_len = copy_rows + self.max_suffix_length;@@ -363,108 +478,104 @@ impl<B: Backend> CacheLayers<B> {}let new_shape = [new_total_len, num_groups, head_dim];- let mut new_keys = context.create_array_zeros(- &new_shape,- dtype,- &format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),- );- let mut new_values = context.create_array_zeros(- &new_shape,- dtype,- &format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),- );-- if copy_rows > 0 {- new_keys.copy_slice(&layer.keys, 0, 0..copy_rows, 0);- new_values.copy_slice(&layer.values, 0, 0..copy_rows, 0);- }+ let new_bytes = size_for_shape(&new_shape, dtype);+ let new_keys = context+ .create_allocation(new_bytes, AllocationType::Global)+ .expect("Failed to create kv keys clone allocation");+ let new_values = context+ .create_allocation(new_bytes, AllocationType::Global)+ .expect("Failed to create kv values clone allocation");CacheLayer::Transformer(KVCacheLayer {state: layer.state.clone(),keys: new_keys,values: new_values,+ shape: new_shape,+ data_type: dtype,})},CacheLayer::StateSpace(layer) => {- let conv_shape = layer.conv_state.shape().to_vec();- let conv_dtype = layer.conv_state.data_type();- let mut new_conv = context.create_array_uninitialized(- &conv_shape,- conv_dtype,- &format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),- );- new_conv.copy_from_array(&layer.conv_state);-- let ssm_shape = layer.ssm_state.shape().to_vec();- let ssm_dtype = layer.ssm_state.data_type();- let mut new_ssm = context.create_array_uninitialized(- &ssm_shape,- ssm_dtype,- &format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),- );- new_ssm.copy_from_array(&layer.ssm_state);+ let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);+ let ssm_bytes = size_for_shape(&layer.ssm_shape, layer.data_type);+ let new_conv = (conv_bytes > 0).then(|| {+ context+ .create_allocation(conv_bytes, AllocationType::Global)+ .expect("Failed to create ssm conv clone allocation")+ });+ let new_ssm = context+ .create_allocation(ssm_bytes, AllocationType::Global)+ .expect("Failed to create ssm state clone allocation");CacheLayer::StateSpace(SSMLayer {conv_state: new_conv,+ conv_shape: layer.conv_shape,ssm_state: new_ssm,+ ssm_shape: layer.ssm_shape,+ data_type: layer.data_type,})},CacheLayer::ShortConv(layer) => {- let conv_shape = layer.conv_state.shape().to_vec();- let conv_dtype = layer.conv_state.data_type();- let mut new_conv = context.create_array_uninitialized(- &conv_shape,- conv_dtype,- &format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),- );- new_conv.copy_from_array(&layer.conv_state);-- let suffix_shape = layer.suffix_state.shape().to_vec();- let suffix_dtype = layer.suffix_state.data_type();- let new_suffix = context.create_array_zeros(- &suffix_shape,- suffix_dtype,- &format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),- );+ let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);+ let suffix_bytes = size_for_shape(&layer.suffix_shape, layer.data_type);+ let new_conv = context+ .create_allocation(conv_bytes, AllocationType::Global)+ .expect("Failed to create short conv clone allocation");+ let new_suffix = context+ .create_allocation(suffix_bytes, AllocationType::Global)+ .expect("Failed to create short conv suffix clone allocation");CacheLayer::ShortConv(ShortConvLayer {conv_state: new_conv,+ conv_shape: layer.conv_shape,suffix_state: new_suffix,+ suffix_shape: layer.suffix_shape,+ data_type: layer.data_type,suffix_state_valid_start: Cell::new(0),suffix_state_valid_len: Cell::new(0),})},CacheLayer::DeltaNet(layer) => {- let conv_shape = layer.conv_state.shape().to_vec();- let conv_dtype = layer.conv_state.data_type();- let mut new_conv = context.create_array_uninitialized(- &conv_shape,- conv_dtype,- &format!("{ARRAY_DELTA_NET_CONV_STATE_LABEL}_{layer_index}"),- );- new_conv.copy_from_array(&layer.conv_state);-- let ssm_shape = layer.ssm_state.shape().to_vec();- let ssm_dtype = layer.ssm_state.data_type();- let mut new_ssm = context.create_array_uninitialized(- &ssm_shape,- ssm_dtype,- &format!("{ARRAY_DELTA_NET_SSM_STATE_LABEL}_{layer_index}"),- );- new_ssm.copy_from_array(&layer.ssm_state);+ let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);+ let ssm_bytes = size_for_shape(&layer.ssm_shape, layer.data_type);+ let new_conv = context+ .create_allocation(conv_bytes, AllocationType::Global)+ .expect("Failed to create delta net conv clone allocation");+ let new_ssm = context+ .create_allocation(ssm_bytes, AllocationType::Global)+ .expect("Failed to create delta net ssm clone allocation");CacheLayer::DeltaNet(DeltaNetLayer {conv_state: new_conv,+ conv_shape: layer.conv_shape,ssm_state: new_ssm,+ ssm_shape: layer.ssm_shape,+ data_type: layer.data_type,})},}).collect();- Self {+ let mut zero_encoder: Encoder<B> = Encoder::new(context).expect("Failed to create cache clone zero encoder");+ for layer in data.iter_mut() {+ match layer {+ CacheLayer::Transformer(layer) => {+ zero_encoder.encode_fill(&mut layer.keys, 0);+ zero_encoder.encode_fill(&mut layer.values, 0);+ },+ CacheLayer::ShortConv(layer) => {+ zero_encoder.encode_fill(&mut layer.suffix_state, 0);+ },+ _ => {},+ }+ }+ zero_encoder.end_encoding().submit().wait_until_completed().expect("Failed to zero cloned cache layers");++ let mut cloned = Self {max_suffix_length: self.max_suffix_length,max_prefix_length: max_prefix_capacity_across_layers,data,- }+ };+ cloned.copy_from(self, context);+ cloned}}diff --git a/crates/backend-uzu/src/forward_pass/delta_net_layer.rs b/crates/backend-uzu/src/forward_pass/delta_net_layer.rsindex 4559e35cd..3fc31a82e 100644--- a/crates/backend-uzu/src/forward_pass/delta_net_layer.rs+++ b/crates/backend-uzu/src/forward_pass/delta_net_layer.rs@@ -1,16 +1,12 @@-use bytemuck::fill_zeroes;+use crate::{+ DataType,+ backends::common::{Allocation, Backend},+};-use crate::{array::Array, backends::common::Backend};--#[derive(Debug)]pub struct DeltaNetLayer<B: Backend> {- pub conv_state: Array<B>,- pub ssm_state: Array<B>,-}--impl<B: Backend> DeltaNetLayer<B> {- pub fn zero(&mut self) {- fill_zeroes(self.conv_state.as_bytes_mut());- fill_zeroes(self.ssm_state.as_bytes_mut());- }+ pub conv_state: Allocation<B>,+ pub conv_shape: [usize; 2],+ pub ssm_state: Allocation<B>,+ pub ssm_shape: [usize; 3],+ pub data_type: DataType,}diff --git a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rsindex dda8b3b80..a3dec827a 100644--- a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs@@ -1,12 +1,12 @@use crate::{- array::{Array, ArrayContextExt},+ DataType,+ array::{Array, ArrayContextExt, size_for_shape},backends::common::{- Backend, Encoder,+ Allocation, Backend, Encoder, allocation_as_bytes, allocation_as_bytes_mut,kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},},};-#[derive(Clone)]pub enum KVSlice<B: Backend> {Full {base_prefix_len: usize,@@ -16,8 +16,8 @@ pub enum KVSlice<B: Backend> {base_ring_offset: usize,base_ring_length: usize,slots: Vec<usize>,- keys: Array<B>, // [num_groups, slots.len(), head_dim]- values: Array<B>, // [num_groups, slots.len(), head_dim]+ keys: Array<B>, // [slots.len(), num_groups, head_dim]+ values: Array<B>, // [slots.len(), num_groups, head_dim]},}@@ -39,16 +39,60 @@ pub enum KVCacheLayerState {pub const INVALID_POSITION: usize = i32::MAX as usize;-#[derive(Debug)]pub struct KVCacheLayer<B: Backend> {pub state: KVCacheLayerState,- /// [num_groups, max_prefix_length + max_suffix_length, head_dim]- pub keys: Array<B>,- /// [num_groups, max_prefix_length + max_suffix_length, head_dim]- pub values: Array<B>,+ /// [max_prefix_length + max_suffix_length, num_groups, head_dim]+ pub keys: Allocation<B>,+ /// [max_prefix_length + max_suffix_length, num_groups, head_dim]+ pub values: Allocation<B>,+ pub shape: [usize; 3],+ pub data_type: DataType,+}++fn copy_rows(+ source_keys_bytes: &[u8],+ source_values_bytes: &[u8],+ destination_keys_bytes: &mut [u8],+ destination_values_bytes: &mut [u8],+ row_size: usize,+ row_pairs: impl IntoIterator<Item = (usize, usize)>,+) {+ for (source_row, destination_row) in row_pairs {+ let source_offset = source_row * row_size;+ let destination_offset = destination_row * row_size;+ destination_keys_bytes[destination_offset..destination_offset + row_size]+ .copy_from_slice(&source_keys_bytes[source_offset..source_offset + row_size]);+ destination_values_bytes[destination_offset..destination_offset + row_size]+ .copy_from_slice(&source_values_bytes[source_offset..source_offset + row_size]);+ }}impl<B: Backend> KVCacheLayer<B> {+ pub fn encode_copy_prefix_rows_to(+ &self,+ destination: &mut KVCacheLayer<B>,+ row_count: usize,+ encoder: &mut Encoder<B>,+ ) {+ if row_count == 0 {+ return;+ }++ let [source_seq, num_groups, head_dim] = self.shape;+ let [destination_seq, destination_groups, destination_head_dim] = destination.shape;+ assert_eq!(num_groups, destination_groups, "KV cache group count mismatch");+ assert_eq!(head_dim, destination_head_dim, "KV cache head dim mismatch");+ assert_eq!(self.data_type, destination.data_type, "KV cache dtype mismatch");+ assert!(row_count <= source_seq, "source KV cache copy exceeds source sequence");+ assert!(row_count <= destination_seq, "source KV cache copy exceeds destination sequence");++ let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);+ let copy_size = row_count * row_size;++ encoder.encode_copy(&self.keys, 0..copy_size, &mut destination.keys, 0..copy_size);+ encoder.encode_copy(&self.values, 0..copy_size, &mut destination.values, 0..copy_size);+ }+pub fn prefix_segment_length(&self) -> usize {match &self.state {KVCacheLayerState::Full {@@ -139,7 +183,7 @@ impl<B: Backend> KVCacheLayer<B> {}fn scatter_if_required(- &self,+ &mut self,source_indices: &[usize],destination_indices: &[usize],encoder: &mut Encoder<B>,@@ -149,17 +193,15 @@ impl<B: Backend> KVCacheLayer<B> {return;}- let k_shape = self.keys.shape();- let v_shape = self.values.shape();-- let layer_data = KVLayerData {- key_buffer: self.keys.buffer(),- key_shape: [k_shape[0], k_shape[1], k_shape[2]],- value_buffer: self.values.buffer(),- value_shape: [v_shape[0], v_shape[1], v_shape[2]],+ let mut layer_data = KVLayerData {+ key_allocation: &mut self.keys,+ key_shape: self.shape,+ value_allocation: &mut self.values,+ value_shape: self.shape,};- let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, encoder);+ let _ =+ kv_cache_update.encode(std::slice::from_mut(&mut layer_data), source_indices, destination_indices, encoder);}pub fn register_accepted_tokens(@@ -204,16 +246,15 @@ impl<B: Backend> KVCacheLayer<B> {if len == 0 || len > window_length {return None;}- let shape = self.keys.shape();- let num_groups = shape[1];- let head_dim = shape[2];- let dtype = self.keys.data_type();+ let [_, num_groups, head_dim] = self.shape;+ let dtype = self.data_type;+ let row_size = size_for_shape(&[1, num_groups, head_dim], dtype);let slice_shape = [len, num_groups, head_dim];- let mut slice_keys =- context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_keys");- let mut slice_values =- context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_values");+ let mut slice_keys = context.create_array_uninitialized(&slice_shape, dtype);+ let mut slice_values = context.create_array_uninitialized(&slice_shape, dtype);+ let source_keys_bytes = allocation_as_bytes(&self.keys);+ let source_values_bytes = allocation_as_bytes(&self.values);let slots: Vec<usize> = (range.start..range.end).enumerate()@@ -227,10 +268,17 @@ impl<B: Backend> KVCacheLayer<B> {}).collect();- for (i, &slot) in slots.iter().enumerate() {- slice_keys.copy_slice(&self.keys, 0, slot..slot + 1, i);- slice_values.copy_slice(&self.values, 0, slot..slot + 1, i);- }+ let destination_keys_bytes = slice_keys.as_bytes_mut();+ let destination_values_bytes = slice_values.as_bytes_mut();++ copy_rows(+ source_keys_bytes,+ source_values_bytes,+ destination_keys_bytes,+ destination_values_bytes,+ row_size,+ slots.iter().enumerate().map(|(destination_row, &source_row)| (source_row, destination_row)),+ );Some(KVSlice::Window {window_length,@@ -281,15 +329,28 @@ impl<B: Backend> KVCacheLayer<B> {},) => {*w_len = *window_length;+ let [_, num_groups, head_dim] = self.shape;+ let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);+ let source_keys_bytes = keys.as_bytes();+ let source_values_bytes = values.as_bytes();+ let destination_keys_bytes = allocation_as_bytes_mut(&mut self.keys);+ let destination_values_bytes = allocation_as_bytes_mut(&mut self.values);match range {None => {*ring_offset = *base_ring_offset;*ring_length = *base_ring_length;- for (i, &slot) in slots.iter().enumerate() {- self.keys.copy_slice(keys, 0, i..i + 1, slot);- self.values.copy_slice(values, 0, i..i + 1, slot);- }+ copy_rows(+ source_keys_bytes,+ source_values_bytes,+ destination_keys_bytes,+ destination_values_bytes,+ row_size,+ slots+ .iter()+ .enumerate()+ .map(|(source_row, &destination_row)| (source_row, destination_row)),+ );},Some(r) => {let len = r.end.saturating_sub(r.start);@@ -316,11 +377,17 @@ impl<B: Backend> KVCacheLayer<B> {*ring_offset = new_offset;*ring_length = new_len;- for (i, &slot) in slots[r.clone()].iter().enumerate() {- let src_i = r.start + i;- self.keys.copy_slice(keys, 0, src_i..src_i + 1, slot);- self.values.copy_slice(values, 0, src_i..src_i + 1, slot);- }+ copy_rows(+ source_keys_bytes,+ source_values_bytes,+ destination_keys_bytes,+ destination_values_bytes,+ row_size,+ slots[r.clone()]+ .iter()+ .enumerate()+ .map(|(index, &destination_row)| (r.start + index, destination_row)),+ );},}},diff --git a/crates/backend-uzu/src/forward_pass/mod.rs b/crates/backend-uzu/src/forward_pass/mod.rsindex 474f782c6..3f599a5ec 100644--- a/crates/backend-uzu/src/forward_pass/mod.rs+++ b/crates/backend-uzu/src/forward_pass/mod.rs@@ -5,8 +5,8 @@ pub mod delta_net_layer;pub mod kv_cache_layer;pub mod short_conv_layer;pub mod ssm_layer;+pub mod token_inputs;-pub mod scratch_buffers;pub mod state;#[cfg(feature = "tracing")]diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rsindex fc56220d3..d9a47b659 100644--- a/crates/backend-uzu/src/forward_pass/model_shape.rs+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs@@ -3,7 +3,7 @@ use crate::{config::{DecoderConfig, DecoderLayerType, LinearConfig, MLPConfig, MixerConfig},};-#[derive(Debug)]+#[derive(Debug, Clone)]pub struct ModelShape {activation_type: DataType,kv_cache_type: DataType,diff --git a/crates/backend-uzu/src/forward_pass/scratch_buffers.rs b/crates/backend-uzu/src/forward_pass/scratch_buffers.rsdeleted file mode 100644index 2cefb1d1c..000000000--- a/crates/backend-uzu/src/forward_pass/scratch_buffers.rs+++ /dev/null@@ -1,216 +0,0 @@-use crate::{- DataType,- array::{Array, ArrayContextExt},- backends::common::Backend,- config::{DecoderConfig, MLPConfig},- forward_pass::model_shape::ModelShape,-};--pub struct ScratchBuffers<B: Backend> {- // 1-D- pub token_ids: Array<B>,- pub token_subtrie_ranges: Array<B>,- pub token_positions: Array<B>,- pub token_parents: Array<B>,- pub token_bitmask: Array<B>,- pub token_seeds: Array<B>,- pub sampling_output: Array<B>,-- // 2-D- pub logits: Array<B>,- pub main: Array<B>,- pub shortcut: Array<B>,- pub qkv: Array<B>,- pub gate: Option<Array<B>>,- pub attention_output: Array<B>,- pub mlp_fused_up: Array<B>,- pub mlp_hidden: Array<B>,- pub lora_intermediate: Option<Array<B>>,- pub ssm_inproj: Option<Array<B>>,- pub ssm_packed: Option<Array<B>>,- pub ssm_conv_padded: Option<Array<B>>,- pub ssm_x: Option<Array<B>>,- pub ssm_b: Option<Array<B>>,- pub ssm_c: Option<Array<B>>,- pub ssm_dt: Option<Array<B>>,- pub ssm_z: Option<Array<B>>,- pub delta_net_prep_q_norm: Option<Array<B>>,- pub delta_net_prep_k_norm: Option<Array<B>>,- pub delta_net_prep_beta: Option<Array<B>>,- pub delta_net_prep_decay: Option<Array<B>>,-- // 3-D- pub rotated_queries: Array<B>,- pub rotated_keys: Array<B>,- pub extracted_values: Array<B>,-- // 2-pass attention intermediate buffers- pub attention_partials: Array<B>, // [num_heads * max_suffix_len * total_blocks_count * head_dim]- pub attention_sums: Array<B>, // [num_heads * max_suffix_len * total_blocks_count]- pub attention_maxs: Array<B>, // [num_heads * max_suffix_len * total_blocks_count]-- pub moe_topk_ids: Option<Array<B>>,- pub moe_topk_probs: Option<Array<B>>,- pub moe_offsets: Option<Array<B>>,- pub moe_sumk: Option<Array<B>>,- pub moe_bucketed_token_ids: Option<Array<B>>,- pub moe_bucketed_probs: Option<Array<B>>,- pub moe_x_perm: Option<Array<B>>,- pub moe_tok2row: Option<Array<B>>,- pub moe_y_partial: Option<Array<B>>,- pub moe_hidden: Option<Array<B>>,- pub moe_two_pass_row_expert_map: Option<Array<B>>,- pub moe_tile_counts: Option<Array<B>>,- pub moe_tile_offsets: Option<Array<B>>,- pub moe_tile_map: Option<Array<B>>,- pub moe_total_tiles: Option<Array<B>>,- pub moe_dispatch_args: Option<Array<B>>,- pub moe_scatter_partials: Option<Array<B>>,- pub moe_scatter_block_bases: Option<Array<B>>,- pub moe_block_alloc: Option<Array<B>>,-}--impl<B: Backend> ScratchBuffers<B> {- pub fn new(- context: &B::Context,- decoder_config: &DecoderConfig,- model_shape: &ModelShape,- max_suffix_len: usize,- ) -> Self {- // Helper closure for allocation- let alloc = |shape: &[usize], dtype: DataType, label: &str| -> Array<B> {- context.create_array_uninitialized(shape, dtype, &format!("scratch_buffers_{label}"))- };-- let act_ty = model_shape.activation_data_type();-- let sums_maxs_shape = model_shape.attention_sums_shape(max_suffix_len);- let moe = match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => Some(moe),- _ => None,- };- let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_active_routed_experts);- let moe_scatter_entries = moe.map(|moe| {- let num_blocks = ((max_suffix_len + 255) / 256).max(1);- let num_tiles = ((moe.num_routed_experts + 512 - 1) / 512).max(1);- num_blocks * num_tiles * 512- });-- Self {- // 1-D- token_ids: alloc(&[max_suffix_len], DataType::U64, "token_ids"),- token_subtrie_ranges: alloc(- &model_shape.subtrie_ranges_shape(max_suffix_len),- DataType::U32,- "token_subtrie_ranges",- ),- token_positions: alloc(&[max_suffix_len], DataType::I32, "token_positions"),- token_parents: alloc(&[max_suffix_len], DataType::I32, "token_parents"),- token_bitmask: alloc(&model_shape.bitmask_shape(max_suffix_len), DataType::U32, "token_bitmask"),- token_seeds: alloc(&[max_suffix_len], DataType::U64, "token_seeds"),- sampling_output: alloc(&[max_suffix_len], DataType::U32, "sampling_output"),-- // 2-D- logits: alloc(&model_shape.logits_shape(max_suffix_len), act_ty, "logits"),- main: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "main"),- shortcut: alloc(&model_shape.main_shape(max_suffix_len), act_ty, "shortcut"),- qkv: alloc(&model_shape.qkv_shape(max_suffix_len), act_ty, "qkv"),- gate: model_shape.gate_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "gate")),- attention_output: alloc(&model_shape.attention_output_shape(max_suffix_len), act_ty, "attention_output"),- mlp_fused_up: alloc(&model_shape.mlp_fused_up_shape(max_suffix_len), act_ty, "mlp_fused_up"),- mlp_hidden: alloc(&model_shape.mlp_hidden_shape(max_suffix_len), act_ty, "mlp_hidden"),- lora_intermediate: model_shape- .lora_intermediate(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "lora_intermediate")),- ssm_inproj: model_shape.ssm_inproj_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_inproj")),- ssm_packed: model_shape.ssm_packed_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_packed")),- ssm_conv_padded: model_shape- .ssm_conv_padded_shape(max_suffix_len)- .map(|shape| alloc(&shape, act_ty, "ssm_conv_padded")),- ssm_x: model_shape.ssm_x_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_x")),- ssm_b: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_b")),- ssm_c: model_shape.ssm_bc_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_c")),- ssm_dt: model_shape.ssm_dt_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_dt")),- ssm_z: model_shape.ssm_z_shape(max_suffix_len).map(|shape| alloc(&shape, act_ty, "ssm_z")),- delta_net_prep_q_norm: model_shape- .delta_net_prep_qk_shape(max_suffix_len)- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_q_norm")),- delta_net_prep_k_norm: model_shape- .delta_net_prep_qk_shape(max_suffix_len)- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_k_norm")),- delta_net_prep_beta: model_shape- .delta_net_prep_beta_decay_shape(max_suffix_len)- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_beta")),- delta_net_prep_decay: model_shape- .delta_net_prep_beta_decay_shape(max_suffix_len)- .map(|shape| alloc(&shape, DataType::F32, "delta_net_prep_decay")),- // 3-D- rotated_queries: alloc(&model_shape.rotated_queries_shape(max_suffix_len), act_ty, "rotated_queries"),- rotated_keys: alloc(&model_shape.rotated_keys_shape(max_suffix_len), act_ty, "rotated_keys"),- extracted_values: alloc(&model_shape.extracted_values_shape(max_suffix_len), act_ty, "extracted_values"),-- attention_partials: alloc(- &model_shape.attention_partials_shape(max_suffix_len),- act_ty,- "attention_partials",- ),- attention_sums: alloc(&sums_maxs_shape, act_ty, "attention_sums"),- attention_maxs: alloc(&sums_maxs_shape, act_ty, "attention_maxs"),-- moe_topk_ids: moe.map(|moe| {- alloc(- &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_active_routed_experts),- DataType::U32,- "moe_topk_ids",- )- }),- moe_topk_probs: moe.map(|moe| {- alloc(- &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_active_routed_experts),- act_ty,- "moe_topk_probs",- )- }),- moe_offsets: moe- .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_offsets")),- moe_sumk: moe.map(|_| alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")),- moe_bucketed_token_ids: moe_max_routed.map(|max_routed| {- alloc(&model_shape.moe_bucketed_token_ids_shape(max_routed), DataType::U32, "moe_bucketed_token_ids")- }),- moe_bucketed_probs: moe_max_routed.map(|max_routed| {- alloc(&model_shape.moe_bucketed_probs_shape(max_routed), act_ty, "moe_bucketed_probs")- }),- moe_x_perm: moe_max_routed- .map(|max_routed| alloc(&model_shape.moe_x_perm_shape(max_routed), DataType::F16, "moe_x_perm")),- moe_tok2row: moe.map(|moe| {- alloc(- &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_active_routed_experts),- DataType::I32,- "moe_tok2row",- )- }),- moe_y_partial: moe_max_routed- .map(|max_routed| alloc(&model_shape.moe_y_partial_shape(max_routed), DataType::F16, "moe_y_partial")),- moe_hidden: moe_max_routed- .map(|max_routed| alloc(&model_shape.moe_hidden_shape(max_routed), DataType::F32, "moe_hidden")),- moe_two_pass_row_expert_map: moe_max_routed- .map(|max_routed| alloc(&[max_routed], DataType::U32, "moe_two_pass_row_expert_map")),- moe_tile_counts: moe.map(|moe| {- alloc(&model_shape.moe_counts_shape(moe.num_routed_experts), DataType::U32, "moe_tile_counts")- }),- moe_tile_offsets: moe.map(|moe| {- alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_tile_offsets")- }),- moe_tile_map: moe_max_routed- .map(|max_routed| alloc(&model_shape.moe_tile_map_shape(max_routed), DataType::U32, "moe_tile_map")),- moe_total_tiles: moe.map(|_| alloc(&model_shape.moe_total_tiles_shape(), DataType::U32, "moe_total_tiles")),- moe_dispatch_args: moe- .map(|_| alloc(&model_shape.moe_dispatch_args_shape(), DataType::U32, "moe_dispatch_args")),- moe_scatter_partials: moe_scatter_entries- .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_partials")),- moe_scatter_block_bases: moe_scatter_entries- .map(|entries| alloc(&[entries], DataType::U32, "moe_scatter_block_bases")),- moe_block_alloc: moe_scatter_entries.map(|entries| alloc(&[entries], DataType::U32, "moe_block_alloc")),- }- }-}diff --git a/crates/backend-uzu/src/forward_pass/short_conv_layer.rs b/crates/backend-uzu/src/forward_pass/short_conv_layer.rsindex a7f7958e7..dbb7474fc 100644--- a/crates/backend-uzu/src/forward_pass/short_conv_layer.rs+++ b/crates/backend-uzu/src/forward_pass/short_conv_layer.rs@@ -1,16 +1,20 @@use std::cell::Cell;-use bytemuck::fill_zeroes;+use crate::{+ DataType,+ array::size_for_shape,+ backends::common::{Allocation, Backend, Encoder},+};-use crate::{array::Array, backends::common::Backend};--#[derive(Debug)]pub struct ShortConvLayer<B: Backend> {- pub conv_state: Array<B>,+ pub conv_state: Allocation<B>,+ pub conv_shape: [usize; 2],/// Per-token post-state written during speculative runs.////// Shape: [max_suffix_length, model_dim, state_stride]- pub suffix_state: Array<B>,+ pub suffix_state: Allocation<B>,+ pub suffix_shape: [usize; 3],+ pub data_type: DataType,/// Start index (in the suffix batch) for which `suffix_state` contains/// valid post-states for this layer.pub suffix_state_valid_start: Cell<usize>,@@ -20,12 +24,6 @@ pub struct ShortConvLayer<B: Backend> {}impl<B: Backend> ShortConvLayer<B> {- pub fn zero(&mut self) {- fill_zeroes(self.conv_state.as_bytes_mut());- fill_zeroes(self.suffix_state.as_bytes_mut());- self.clear_suffix_state_valid_range();- }-pub fn clear_suffix_state_valid_range(&self) {self.suffix_state_valid_start.set(0);self.suffix_state_valid_len.set(0);@@ -43,6 +41,7 @@ impl<B: Backend> ShortConvLayer<B> {pub fn commit_from_suffix_state_if_valid(&mut self,commit_index: usize,+ encoder: &mut Encoder<B>,) {let start = self.suffix_state_valid_start.get();let len = self.suffix_state_valid_len.get();@@ -53,38 +52,19 @@ impl<B: Backend> ShortConvLayer<B> {return;}- assert_eq!(- self.conv_state.data_type(),- self.suffix_state.data_type(),- "ShortConv conv_state / suffix_state dtype mismatch"- );-- let [model_dim, state_stride] = {- let shape = self.conv_state.shape();- assert_eq!(shape.len(), 2, "ShortConv conv_state expected 2-D [model_dim, state_stride], got {:?}", shape);- [shape[0], shape[1]]- };+ let [model_dim, state_stride] = self.conv_shape;+ let [suffix_length, suffix_model_dim, suffix_stride] = self.suffix_shape;+ assert_eq!(suffix_model_dim, model_dim, "ShortConv suffix_state model_dim mismatch");+ assert_eq!(suffix_stride, state_stride, "ShortConv suffix_state stride mismatch");+ assert!(commit_index < suffix_length, "ShortConv commit_index {} out of range {}", commit_index, suffix_length);- let suffix_shape = self.suffix_state.shape();- assert!(- suffix_shape.len() == 3 && suffix_shape[1] == model_dim && suffix_shape[2] == state_stride,- "ShortConv suffix_state expected 3-D [suffix_len, model_dim, state_stride], got {:?}",- suffix_shape- );- assert!(- commit_index < suffix_shape[0],- "ShortConv commit_index {} out of suffix_state range {}",- commit_index,- suffix_shape[0]+ let bytes_per_token = size_for_shape(&[model_dim, state_stride], self.data_type);+ let src_start = commit_index * bytes_per_token;+ encoder.encode_copy(+ &self.suffix_state,+ src_start..src_start + bytes_per_token,+ &mut self.conv_state,+ 0..bytes_per_token,);-- let elem_bytes = self.conv_state.data_type().size_in_bytes();- let bytes_per_token = model_dim.saturating_mul(state_stride).saturating_mul(elem_bytes);- let src_start = commit_index.saturating_mul(bytes_per_token);- let src_end = src_start.saturating_add(bytes_per_token);-- let src = &self.suffix_state.as_bytes()[src_start..src_end];- let dst = self.conv_state.as_bytes_mut();- dst.copy_from_slice(src);}}diff --git a/crates/backend-uzu/src/forward_pass/ssm_layer.rs b/crates/backend-uzu/src/forward_pass/ssm_layer.rsindex 807a65762..d7733d5d6 100644--- a/crates/backend-uzu/src/forward_pass/ssm_layer.rs+++ b/crates/backend-uzu/src/forward_pass/ssm_layer.rs@@ -1,16 +1,12 @@-use bytemuck::fill_zeroes;+use crate::{+ DataType,+ backends::common::{Allocation, Backend},+};-use crate::{array::Array, backends::common::Backend};--#[derive(Debug)]pub struct SSMLayer<B: Backend> {- pub conv_state: Array<B>,- pub ssm_state: Array<B>,-}--impl<B: Backend> SSMLayer<B> {- pub fn zero(&mut self) {- fill_zeroes(self.conv_state.as_bytes_mut());- fill_zeroes(self.ssm_state.as_bytes_mut());- }+ pub conv_state: Option<Allocation<B>>,+ pub conv_shape: [usize; 2],+ pub ssm_state: Allocation<B>,+ pub ssm_shape: [usize; 3],+ pub data_type: DataType,}diff --git a/crates/backend-uzu/src/forward_pass/state/array_id.rs b/crates/backend-uzu/src/forward_pass/state/array_id.rsdeleted file mode 100644index 281b1470b..000000000--- a/crates/backend-uzu/src/forward_pass/state/array_id.rs+++ /dev/null@@ -1,76 +0,0 @@-use serde::{Deserialize, Serialize};--use super::RopeType;--#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]-pub enum ArrayId {- TokenIds,- TokenPositions,- TokenParents,- TokenBitmask,- Logits,- TokenSeeds,-- Main,- Shortcut,- QKV,- Gate,- AttentionOutput,- MlpFusedUp,- MlpHidden,- SsmInProj,-- Keys(usize),- Values(usize),- SsmConvState(usize),- SsmState(usize),- SsmPacked(usize),- SsmX(usize),- SsmB(usize),- SsmC(usize),- SsmDt(usize),- SsmZ(usize),- ShortConvState(usize),- ShortConvSuffixState(usize),- DeltaNetConvState(usize),- DeltaNetSsmState(usize),-- RotatedQueries,- RotatedKeys,- ExtractedValues,-- AttentionPartials,- AttentionSums,- AttentionMaxs,-- RopeCosines(RopeType),- RopeSines(RopeType),-- AttentionSinks(usize),-- MoeTopkIds,- MoeTopkProbs,- MoeOffsets,- MoeSumK,- MoeBucketedTokenIds,- MoeBucketedProbs,- MoeXPerm,- MoeTok2Row,- MoeYPartial,- MoeHidden,- MoeTwoPassRowExpertMap,- MoeTileCounts,- MoeTileOffsets,- MoeTileMap,- MoeTotalTiles,- MoeDispatchArgs,- MoeScatterPartials,- MoeScatterBlockBases,- MoeBlockAlloc,-- // Classifier prediction head buffers- ClassifierPooling,- ClassifierPredictionHeadDense,- ClassifierPredictionHeadNorm,- ClassifierPredictionHeadLogits,-}diff --git a/crates/backend-uzu/src/forward_pass/state/common_aux_buffers.rs b/crates/backend-uzu/src/forward_pass/state/common_aux_buffers.rsdeleted file mode 100644index 2124f53a6..000000000--- a/crates/backend-uzu/src/forward_pass/state/common_aux_buffers.rs+++ /dev/null@@ -1,53 +0,0 @@-use crate::{- array::Array,- backends::common::Backend,- forward_pass::{model_shape::ModelShape, scratch_buffers::ScratchBuffers},-};--pub struct CommonAuxBuffers<B: Backend> {- pub suffix_length: usize,- pub main: Array<B>,- pub shortcut: Array<B>,- pub qkv: Array<B>,- pub gate: Option<Array<B>>,- pub attention_output: Array<B>,- pub mlp_fused_up: Array<B>,- pub mlp_hidden: Array<B>,- pub lora_intermediate: Option<Array<B>>,- pub rotated_queries: Array<B>,- pub rotated_keys: Array<B>,- pub extracted_values: Array<B>,- pub attention_partials: Array<B>,- pub attention_sums: Array<B>,- pub attention_maxs: Array<B>,-}--impl<B: Backend> CommonAuxBuffers<B> {- pub fn new(- scratch: &ScratchBuffers<B>,- model_shape: &ModelShape,- suffix_length: usize,- ) -> Self {- Self {- suffix_length,- main: scratch.main.view(&model_shape.main_shape(suffix_length)),- shortcut: scratch.shortcut.view(&model_shape.main_shape(suffix_length)),- qkv: scratch.qkv.view(&model_shape.qkv_shape(suffix_length)),- gate: scratch.gate.as_ref().zip(model_shape.gate_shape(suffix_length)).map(|(buf, shape)| buf.view(&shape)),- attention_output: scratch.attention_output.view(&model_shape.attention_output_shape(suffix_length)),- mlp_fused_up: scratch.mlp_fused_up.view(&model_shape.mlp_fused_up_shape(suffix_length)),- mlp_hidden: scratch.mlp_hidden.view(&model_shape.mlp_hidden_shape(suffix_length)),- lora_intermediate: scratch- .lora_intermediate- .as_ref()- .zip(model_shape.lora_intermediate(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- rotated_queries: scratch.rotated_queries.view(&model_shape.rotated_queries_shape(suffix_length)),- rotated_keys: scratch.rotated_keys.view(&model_shape.rotated_keys_shape(suffix_length)),- extracted_values: scratch.extracted_values.view(&model_shape.extracted_values_shape(suffix_length)),- attention_partials: scratch.attention_partials.view(&model_shape.attention_partials_shape(suffix_length)),- attention_sums: scratch.attention_sums.view(&model_shape.attention_sums_shape(suffix_length)),- attention_maxs: scratch.attention_maxs.view(&model_shape.attention_maxs_shape(suffix_length)),- }- }-}diff --git a/crates/backend-uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/backend-uzu/src/forward_pass/state/language_model_generator_aux_buffers.rsdeleted file mode 100644index 09a974544..000000000--- a/crates/backend-uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs+++ /dev/null@@ -1,178 +0,0 @@-use crate::{- array::Array,- backends::common::Backend,- config::{DecoderConfig, MLPConfig},- forward_pass::{model_shape::ModelShape, scratch_buffers::ScratchBuffers},-};--pub struct LanguageModelGeneratorAuxBuffers<B: Backend> {- pub ssm_inproj: Option<Array<B>>,- pub ssm_packed: Option<Array<B>>,- pub ssm_conv_padded: Option<Array<B>>,- pub ssm_x: Option<Array<B>>,- pub ssm_b: Option<Array<B>>,- pub ssm_c: Option<Array<B>>,- pub ssm_dt: Option<Array<B>>,- pub ssm_z: Option<Array<B>>,- // DeltaNet prep buffers- pub delta_net_prep_q_norm: Option<Array<B>>,- pub delta_net_prep_k_norm: Option<Array<B>>,- pub delta_net_prep_beta: Option<Array<B>>,- pub delta_net_prep_decay: Option<Array<B>>,- // MoE buffers- pub moe_topk_ids: Option<Array<B>>,- pub moe_topk_probs: Option<Array<B>>,- pub moe_offsets: Option<Array<B>>,- pub moe_sumk: Option<Array<B>>,- pub moe_bucketed_token_ids: Option<Array<B>>,- pub moe_bucketed_probs: Option<Array<B>>,- pub moe_x_perm: Option<Array<B>>,- pub moe_tok2row: Option<Array<B>>,- pub moe_y_partial: Option<Array<B>>,- pub moe_hidden: Option<Array<B>>,- pub moe_two_pass_row_expert_map: Option<Array<B>>,- pub moe_tile_counts: Option<Array<B>>,- pub moe_tile_offsets: Option<Array<B>>,- pub moe_tile_map: Option<Array<B>>,- pub moe_total_tiles: Option<Array<B>>,- pub moe_dispatch_args: Option<Array<B>>,- pub moe_scatter_partials: Option<Array<B>>,- pub moe_scatter_block_bases: Option<Array<B>>,- pub moe_block_alloc: Option<Array<B>>,-}--impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {- pub fn new(- scratch: &ScratchBuffers<B>,- decoder_config: &DecoderConfig,- model_shape: &ModelShape,- suffix_length: usize,- ) -> Self {- let moe = match &decoder_config.layer_config.mlp_config {- MLPConfig::MixtureOfExperts(moe) => Some(moe),- _ => None,- };- let max_routed = moe.map(|moe| suffix_length * moe.num_active_routed_experts);- let num_blocks = suffix_length.div_ceil(256).max(1);- let scatter_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1) * 512);- let block_alloc_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1));-- Self {- ssm_inproj: scratch- .ssm_inproj- .as_ref()- .zip(model_shape.ssm_inproj_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- ssm_packed: scratch- .ssm_packed- .as_ref()- .zip(model_shape.ssm_packed_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- ssm_conv_padded: scratch- .ssm_conv_padded- .as_ref()- .zip(model_shape.ssm_conv_padded_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- ssm_x: scratch- .ssm_x- .as_ref()- .zip(model_shape.ssm_x_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- ssm_b: scratch- .ssm_b- .as_ref()- .zip(model_shape.ssm_bc_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- ssm_c: scratch- .ssm_c- .as_ref()- .zip(model_shape.ssm_bc_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- ssm_dt: scratch- .ssm_dt- .as_ref()- .zip(model_shape.ssm_dt_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- ssm_z: scratch- .ssm_z- .as_ref()- .zip(model_shape.ssm_z_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- delta_net_prep_q_norm: scratch- .delta_net_prep_q_norm- .as_ref()- .zip(model_shape.delta_net_prep_qk_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- delta_net_prep_k_norm: scratch- .delta_net_prep_k_norm- .as_ref()- .zip(model_shape.delta_net_prep_qk_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- delta_net_prep_beta: scratch- .delta_net_prep_beta- .as_ref()- .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- delta_net_prep_decay: scratch- .delta_net_prep_decay- .as_ref()- .zip(model_shape.delta_net_prep_beta_decay_shape(suffix_length))- .map(|(buf, shape)| buf.view(&shape)),- moe_topk_ids: moe.zip(scratch.moe_topk_ids.as_ref()).map(|(moe, buf)| {- buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_active_routed_experts))- }),- moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(|(moe, buf)| {- buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_active_routed_experts))- }),- moe_offsets: moe- .zip(scratch.moe_offsets.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),- moe_sumk: moe.and(scratch.moe_sumk.as_ref()).map(|buf| buf.view(&model_shape.moe_sumk_shape())),- moe_bucketed_token_ids: max_routed- .zip(scratch.moe_bucketed_token_ids.as_ref())- .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_token_ids_shape(max_routed))),- moe_bucketed_probs: max_routed- .zip(scratch.moe_bucketed_probs.as_ref())- .map(|(max_routed, buf)| buf.view(&model_shape.moe_bucketed_probs_shape(max_routed))),- moe_x_perm: max_routed- .zip(scratch.moe_x_perm.as_ref())- .map(|(max_routed, buf)| buf.view(&model_shape.moe_x_perm_shape(max_routed))),- moe_tok2row: moe.zip(scratch.moe_tok2row.as_ref()).map(|(moe, buf)| {- buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_active_routed_experts))- }),- moe_y_partial: max_routed- .zip(scratch.moe_y_partial.as_ref())- .map(|(max_routed, buf)| buf.view(&model_shape.moe_y_partial_shape(max_routed))),- moe_hidden: max_routed- .zip(scratch.moe_hidden.as_ref())- .map(|(max_routed, buf)| buf.view(&model_shape.moe_hidden_shape(max_routed))),- moe_two_pass_row_expert_map: max_routed- .zip(scratch.moe_two_pass_row_expert_map.as_ref())- .map(|(max_routed, buf)| buf.view(&[max_routed])),- moe_tile_counts: moe- .zip(scratch.moe_tile_counts.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.num_routed_experts))),- moe_tile_offsets: moe- .zip(scratch.moe_tile_offsets.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),- moe_tile_map: max_routed- .zip(scratch.moe_tile_map.as_ref())- .map(|(max_routed, buf)| buf.view(&model_shape.moe_tile_map_shape(max_routed))),- moe_total_tiles: moe- .and(scratch.moe_total_tiles.as_ref())- .map(|buf| buf.view(&model_shape.moe_total_tiles_shape())),- moe_dispatch_args: moe- .and(scratch.moe_dispatch_args.as_ref())- .map(|buf| buf.view(&model_shape.moe_dispatch_args_shape())),- moe_scatter_partials: scatter_entries- .zip(scratch.moe_scatter_partials.as_ref())- .map(|(entries, buf)| buf.view(&[entries])),- moe_scatter_block_bases: scatter_entries- .zip(scratch.moe_scatter_block_bases.as_ref())- .map(|(entries, buf)| buf.view(&[entries])),- moe_block_alloc: block_alloc_entries- .zip(scratch.moe_block_alloc.as_ref())- .map(|(entries, buf)| buf.view(&[entries])),- }- }-}diff --git a/crates/backend-uzu/src/forward_pass/state/mod.rs b/crates/backend-uzu/src/forward_pass/state/mod.rsindex 531093517..efa4164c8 100644--- a/crates/backend-uzu/src/forward_pass/state/mod.rs+++ b/crates/backend-uzu/src/forward_pass/state/mod.rs@@ -1,17 +1,8 @@-mod array_id;mod rope_type;-mod common_aux_buffers;-mod language_model_generator_aux_buffers;mod rope_buffers;mod shared_buffers;-mod state;--pub use array_id::ArrayId;-pub use common_aux_buffers::CommonAuxBuffers;-pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;pub use rope_buffers::RopeBuffers;pub use rope_type::RopeType;pub use shared_buffers::SharedBuffers;-pub use state::ForwardPassState;diff --git a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rsindex 39de3ab9c..37894d4a7 100644--- a/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs+++ b/crates/backend-uzu/src/forward_pass/state/rope_buffers.rs@@ -1,15 +1,15 @@use crate::{- array::{Array, ArrayContextExt},- backends::common::Backend,+ array::ArrayContextExt,+ backends::common::{Allocation, Backend},forward_pass::model_shape::ModelShape,parameters::ParameterTree,};pub struct RopeBuffers<B: Backend> {/// [rope_max_sequence_length, rope_dim]- pub cosines: Array<B>,+ pub cosines: Allocation<B>,/// [rope_max_sequence_length, rope_dim]- pub sines: Array<B>,+ pub sines: Allocation<B>,}impl<B: Backend> RopeBuffers<B> {@@ -21,16 +21,12 @@ impl<B: Backend> RopeBuffers<B> {let rope_max_sequence_length = model_shape.context_length();Self {- cosines: context.create_array_uninitialized(- &[rope_max_sequence_length, rope_dim],- model_shape.activation_data_type(),- "rope_buffers_cosines",- ),- sines: context.create_array_uninitialized(- &[rope_max_sequence_length, rope_dim],- model_shape.activation_data_type(),- "rope_buffers_sines",- ),+ cosines: context+ .create_array_uninitialized(&[rope_max_sequence_length, rope_dim], model_shape.activation_data_type())+ .into_allocation(),+ sines: context+ .create_array_uninitialized(&[rope_max_sequence_length, rope_dim], model_shape.activation_data_type())+ .into_allocation(),}}@@ -43,10 +39,7 @@ impl<B: Backend> RopeBuffers<B> {return;};- let cosines_view = rope_tree.leaf_array("cosines").unwrap();- self.cosines.copy_from_array(&cosines_view);-- let sines_view = rope_tree.leaf_array("sines").unwrap();- self.sines.copy_from_array(&sines_view);+ self.cosines = rope_tree.leaf_allocation("cosines").unwrap();+ self.sines = rope_tree.leaf_allocation("sines").unwrap();}}diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rsindex 2e9e8a49b..6011bbd2c 100644--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs+++ b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs@@ -1,10 +1,10 @@use half::{bf16, f16};-use super::RopeBuffers;+use super::{RopeBuffers, RopeType};use crate::{DataType,- array::{Array, ArrayContextExt},- backends::common::Backend,+ array::ArrayContextExt,+ backends::common::{Allocation, AsBufferRangeMut, Backend, DenseBuffer},config::DecoderConfig,forward_pass::model_shape::ModelShape,parameters::ParameterTree,@@ -13,7 +13,7 @@ use crate::{pub struct SharedBuffers<B: Backend> {pub global_rope: Option<RopeBuffers<B>>,pub local_rope: Option<RopeBuffers<B>>,- pub attention_sinks: Option<Vec<Array<B>>>,+ pub attention_sinks: Option<Vec<Allocation<B>>>,}impl<B: Backend> SharedBuffers<B> {@@ -29,9 +29,7 @@ impl<B: Backend> SharedBuffers<B> {let attention_sinks = decoder_config.layer_config.attention_config().is_some_and(|c| c.has_sinks).then(|| {let num_heads = decoder_config.num_heads;(0..decoder_config.num_layers)- .map(|_| {- context.create_array_uninitialized(&[num_heads], DataType::F32, "shared_buffers_attention_sinks")- })+ .map(|_| context.create_array_uninitialized(&[num_heads], DataType::F32).into_allocation()).collect()});@@ -60,7 +58,14 @@ impl<B: Backend> SharedBuffers<B> {let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();let attn_tree = layer_tree.subtree("mixer").unwrap();let sinks_arr = attn_tree.leaf_array("sinks").unwrap();- let dst_slice = sink_cell.as_slice_mut::<f32>();+ let dst_slice = unsafe {+ let buffer_range = sink_cell.as_buffer_range_mut();+ let range = buffer_range.range();+ std::slice::from_raw_parts_mut(+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start) as *mut f32,+ range.len() / std::mem::size_of::<f32>(),+ )+ };match sinks_arr.data_type() {DataType::F32 => {@@ -86,4 +91,31 @@ impl<B: Backend> SharedBuffers<B> {}}}++ pub fn rope_cosines(+ &self,+ rope_type: RopeType,+ ) -> Option<&Allocation<B>> {+ match rope_type {+ RopeType::Global => self.global_rope.as_ref().map(|rope| &rope.cosines),+ RopeType::Local => self.local_rope.as_ref().map(|rope| &rope.cosines),+ }+ }++ pub fn rope_sines(+ &self,+ rope_type: RopeType,+ ) -> Option<&Allocation<B>> {+ match rope_type {+ RopeType::Global => self.global_rope.as_ref().map(|rope| &rope.sines),+ RopeType::Local => self.local_rope.as_ref().map(|rope| &rope.sines),+ }+ }++ pub fn attention_sinks(+ &self,+ layer_index: usize,+ ) -> Option<&Allocation<B>> {+ self.attention_sinks.as_ref().map(|sinks| &sinks[layer_index])+ }}diff --git a/crates/backend-uzu/src/forward_pass/state/state.rs b/crates/backend-uzu/src/forward_pass/state/state.rsdeleted file mode 100644index 9d3c96645..000000000--- a/crates/backend-uzu/src/forward_pass/state/state.rs+++ /dev/null@@ -1,698 +0,0 @@-#[cfg(feature = "tracing")]-use std::ops::{Deref, DerefMut};-use std::{cell::RefCell, rc::Rc};--use ndarray::ArrayView2;--#[cfg(feature = "tracing")]-use crate::backends::common::Encoder;-#[cfg(feature = "tracing")]-use crate::forward_pass::traces::ActivationTrace;-use crate::{- DataType,- array::Array,- backends::common::{Backend, Context},- config::DecoderConfig,- forward_pass::{- cache_layers::CacheLayers,- model_shape::ModelShape,- scratch_buffers::ScratchBuffers,- state::{ArrayId, CommonAuxBuffers, LanguageModelGeneratorAuxBuffers, RopeType, SharedBuffers},- },- session::parameter::SamplingMethod,-};--pub enum ForwardPassMode<B: Backend> {- LanguageModelGenerator(LanguageModelGeneratorModeState<B>),- Classifier(ClassifierModeState<B>),-}--pub struct LanguageModelGeneratorModeState<B: Backend> {- pub cache_layers: Rc<RefCell<CacheLayers<B>>>,- pub token_seeds: Array<B>,- pub logits: Array<B>,- pub sampling_output: Option<Array<B>>,- pub sampling_method: Option<SamplingMethod>,- #[cfg(feature = "tracing")]- pub traces: Rc<RefCell<ActivationTrace<B>>>,- pub active_row_count: usize,- pub sampling_start: usize,- pub sampling_length: usize,- pub is_prefilling: bool,-}--pub struct ClassifierModeState<B: Backend> {- pub pooling: Array<B>,- pub dense: Array<B>,- pub norm: Array<B>,- pub classifier_logits: Array<B>,- pub active_row_count: usize,- #[cfg(feature = "tracing")]- pub traces: Rc<RefCell<ActivationTrace<B>>>,-}--pub struct ForwardPassState<B: Backend> {- context: Rc<B::Context>,- token_ids: Array<B>,- pub token_subtrie_ranges: Option<Array<B>>,- token_positions: Array<B>,- token_parents: Array<B>,- token_bitmask: Option<Array<B>>,- pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- pub common_aux: CommonAuxBuffers<B>,- pub llm_aux: Option<LanguageModelGeneratorAuxBuffers<B>>,- mode: ForwardPassMode<B>,-}--impl<B: Backend> ForwardPassState<B> {- // ========================================================================- // Common initialization helpers- // ========================================================================-- fn init_token_ids(- scratch: &ScratchBuffers<B>,- token_ids: &[u64],- ) -> Array<B> {- let suffix_length = token_ids.len();- let mut token_ids_array = scratch.token_ids.view(&[suffix_length]);- token_ids_array.copy_from_view(token_ids.into());- token_ids_array- }-- fn init_token_positions(- scratch: &ScratchBuffers<B>,- token_positions: &[usize],- ) -> Array<B> {- let suffix_length = token_positions.len();- let mut token_positions_array = scratch.token_positions.view(&[suffix_length]);- let token_positions_i32: Box<[i32]> = token_positions.iter().map(|p| *p as i32).collect();- token_positions_array.copy_from_view(token_positions_i32.as_ref().into());- token_positions_array- }-- fn init_token_parents(- scratch: &ScratchBuffers<B>,- token_positions: &[usize],- sampling_start: usize,- sampling_length: usize,- ) -> Array<B> {- let suffix_length = token_positions.len();- let mut token_parents_array = scratch.token_parents.view(&[suffix_length]);- {- let parents = token_parents_array.as_slice_mut::<i32>();- parents.fill(-1);-- if sampling_length > 0 {- let root_pos = token_positions.get(sampling_start).copied().unwrap_or(0);-- let mut stack: Vec<usize> = Vec::new();-- for local_idx in 0..sampling_length {- let abs_idx = sampling_start + local_idx;- let Some(&pos) = token_positions.get(abs_idx) else {- break;- };- let depth = pos.saturating_sub(root_pos);-- let parent_local: i32 = if depth == 0 {- -1- } else if let Some(&p) = stack.get(depth - 1) {- p as i32- } else {- debug_assert!(false, "invalid trie depth ordering: depth={depth}, stack_len={}", stack.len());- -1- };- parents[abs_idx] = parent_local;-- if stack.len() <= depth {- stack.resize(depth + 1, 0);- }- stack[depth] = local_idx;- stack.truncate(depth + 1);- }- }- }- token_parents_array- }-- // ========================================================================- // LLM Constructor- // ========================================================================-- #[allow(clippy::too_many_arguments)]- pub fn new_llm(- context: Rc<B::Context>,- decoder_config: &DecoderConfig,- model_shape: &ModelShape,- scratch: &ScratchBuffers<B>,- cache_layers: Rc<RefCell<CacheLayers<B>>>,- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- token_ids: &[u64],- token_subtrie_ranges: Option<&[[u32; 3]]>,- token_positions: &[usize],- token_bitmask: Option<&[u32]>,- token_seeds: &[u64],- active_row_count: usize,- sampling_start: usize,- sampling_length: usize,- is_prefilling: bool,- skip_token_ids_copy: bool,- async_positions: Option<(Rc<RefCell<B::DenseBuffer>>, usize)>,- async_seeds: Option<(Rc<RefCell<B::DenseBuffer>>, usize)>,- ) -> Self {- let suffix_length = token_ids.len();- assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");- assert!(suffix_length <= cache_layers.borrow().max_suffix_length(), "Suffix length exceeds KV cache capacity");-- // Token IDs - optionally skip copy for async path- let token_ids_cell = if skip_token_ids_copy {- scratch.token_ids.view(&[suffix_length])- } else {- Self::init_token_ids(scratch, token_ids)- };-- // Token subtrie ranges- let token_subtrie_ranges_cell = token_subtrie_ranges.map(|subtrie_ranges| {- let subtrie_ranges_shape = model_shape.subtrie_ranges_shape(suffix_length);- let mut subtrie_ranges_array = scratch.token_subtrie_ranges.view(&subtrie_ranges_shape);- subtrie_ranges_array.as_slice_mut::<u32>().fill(0);- subtrie_ranges_array.copy_from_view(- ArrayView2::from_shape((subtrie_ranges.len(), 3), bytemuck::cast_slice::<_, u32>(subtrie_ranges))- .unwrap(),- );- subtrie_ranges_array- });-- // Token positions - use async buffer if provided- let token_positions_cell = if let Some((async_buf, offset)) = async_positions {- unsafe {- Array::from_parts(async_buf, offset * std::mem::size_of::<i32>(), &[suffix_length], DataType::I32)- }- } else {- Self::init_token_positions(scratch, token_positions)- };-- // Trie parent indices (relative, within the sampling segment).- // Only meaningful when sampling_length > 0.- let token_parents_cell = Self::init_token_parents(scratch, token_positions, sampling_start, sampling_length);-- // Token bitmask- let token_bitmask_cell = token_bitmask.map(|bitmask| {- let bitmask_shape = model_shape.bitmask_shape(suffix_length);- let mut bitmask_array = scratch.token_bitmask.view(&bitmask_shape);- bitmask_array.as_slice_mut::<u32>().fill(0);- bitmask_array.copy_from_view(bitmask.into());- bitmask_array- });-- // Token seeds - use async buffer if provided- let token_seeds_cell = if let Some((async_buf, offset)) = async_seeds {- unsafe {- Array::from_parts(async_buf, offset * std::mem::size_of::<u64>(), &[suffix_length], DataType::U64)- }- } else {- let mut token_seeds_array = scratch.token_seeds.view(&[suffix_length]);- token_seeds_array.copy_from_view(token_seeds.into());- token_seeds_array- };-- // Logits- let logits_cell = scratch.logits.view(&model_shape.logits_shape(suffix_length));-- // Sampling output- let sampling_output = Some(scratch.sampling_output.view(&[suffix_length]));-- // Common aux buffers- let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);-- // LLM-specific aux buffers- let llm_aux = Some(LanguageModelGeneratorAuxBuffers::new(scratch, decoder_config, model_shape, suffix_length));-- // Traces- #[cfg(feature = "tracing")]- let traces = Rc::new(RefCell::new(ActivationTrace::new_llm(context.as_ref(), model_shape, suffix_length)));-- let mode = ForwardPassMode::LanguageModelGenerator(LanguageModelGeneratorModeState {- cache_layers,- token_seeds: token_seeds_cell,- logits: logits_cell,- sampling_output,- sampling_method: None,- #[cfg(feature = "tracing")]- traces,- active_row_count,- sampling_start,- sampling_length,- is_prefilling,- });- Self {- context,- token_ids: token_ids_cell,- token_subtrie_ranges: token_subtrie_ranges_cell,- token_positions: token_positions_cell,- token_parents: token_parents_cell,- token_bitmask: token_bitmask_cell,- shared_buffers,- common_aux,- llm_aux,- mode,- }- }-- // ========================================================================- // Classifier Constructor- // ========================================================================-- #[allow(clippy::too_many_arguments)]- pub fn new_classifier(- context: Rc<B::Context>,- model_shape: &ModelShape,- scratch: &ScratchBuffers<B>,- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- token_ids: &[u64],- token_positions: &[usize],- num_labels: usize,- ) -> Self {- let suffix_length = token_ids.len();- assert_eq!(suffix_length, token_positions.len());-- let token_ids_cell = Self::init_token_ids(scratch, token_ids);- let token_positions_cell = Self::init_token_positions(scratch, token_positions);- let token_parents_cell = Self::init_token_parents(scratch, token_positions, 0, 0);-- // Common aux buffers- let common_aux = CommonAuxBuffers::new(scratch, model_shape, suffix_length);-- // Classifier-specific buffers- let model_dim = model_shape.main_shape(1)[1];- let classifier_state =- Self::init_classifier_buffers(context.as_ref(), model_shape, model_dim, num_labels, suffix_length);-- let mode = ForwardPassMode::Classifier(classifier_state);-- Self {- context,- token_ids: token_ids_cell,- token_subtrie_ranges: None,- token_positions: token_positions_cell,- token_parents: token_parents_cell,- token_bitmask: None,- shared_buffers,- common_aux,- llm_aux: None,- mode,- }- }-- #[cfg(feature = "tracing")]- fn init_classifier_buffers(- context: &B::Context,- model_shape: &ModelShape,- model_dim: usize,- num_labels: usize,- suffix_length: usize,- ) -> ClassifierModeState<B> {- let data_type = model_shape.activation_data_type();- let batch_dim = 1;-- let create_buffer = |size: usize| -> Array<B> {- let buffer_size = size * data_type.size_in_bytes();- let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, size / batch_dim], data_type) }- };-- ClassifierModeState {- pooling: create_buffer(batch_dim * model_dim),- dense: create_buffer(batch_dim * model_dim),- norm: create_buffer(batch_dim * model_dim),- classifier_logits: {- let buffer_size = batch_dim * num_labels * data_type.size_in_bytes();- let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_dim, num_labels], data_type) }- },- active_row_count: suffix_length,- traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(- context,- model_shape,- suffix_length,- num_labels,- ))),- }- }-- #[cfg(not(feature = "tracing"))]- fn init_classifier_buffers(- context: &B::Context,- model_shape: &ModelShape,- model_dim: usize,- num_labels: usize,- suffix_length: usize,- ) -> ClassifierModeState<B> {- let data_type = model_shape.activation_data_type();- let batch_dim = 1;-- let create_buffer = |dims: &[usize]| -> Array<B> {- let size: usize = dims.iter().product();- let buffer_size = size * data_type.size_in_bytes();- let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, dims, data_type) }- };-- ClassifierModeState {- pooling: create_buffer(&[batch_dim, model_dim]),- dense: create_buffer(&[batch_dim, model_dim]),- norm: create_buffer(&[batch_dim, model_dim]),- classifier_logits: create_buffer(&[batch_dim, num_labels]),- active_row_count: suffix_length,- }- }-- // ========================================================================- // Accessors- // ========================================================================-- pub fn context(&self) -> &B::Context {- self.context.as_ref()- }-- pub fn aux_buffers_suffix_length(&self) -> usize {- self.common_aux.suffix_length- }-- pub fn token_bitmask(&self) -> Option<&Array<B>> {- self.token_bitmask.as_ref()- }-- pub fn llm_state(&self) -> &LanguageModelGeneratorModeState<B> {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state,- _ => panic!("Not in LLM mode"),- }- }-- pub fn classifier_state(&self) -> &ClassifierModeState<B> {- match &self.mode {- ForwardPassMode::Classifier(state) => state,- _ => panic!("Not in classifier mode"),- }- }-- pub fn array(- &self,- id: ArrayId,- ) -> Array<B> {- match id {- // Common arrays- ArrayId::TokenIds => self.token_ids.clone(),- ArrayId::TokenPositions => self.token_positions.clone(),- ArrayId::TokenParents => self.token_parents.clone(),- ArrayId::TokenBitmask => self.token_bitmask.clone().expect("Token bitmask not available"),- ArrayId::Main => self.common_aux.main.clone(),- ArrayId::Shortcut => self.common_aux.shortcut.clone(),- ArrayId::QKV => self.common_aux.qkv.clone(),- ArrayId::Gate => self.common_aux.gate.clone().expect("Gate buffer not available"),- ArrayId::AttentionOutput => self.common_aux.attention_output.clone(),- ArrayId::MlpFusedUp => self.common_aux.mlp_fused_up.clone(),- ArrayId::MlpHidden => self.common_aux.mlp_hidden.clone(),- ArrayId::RotatedQueries => self.common_aux.rotated_queries.clone(),- ArrayId::RotatedKeys => self.common_aux.rotated_keys.clone(),- ArrayId::ExtractedValues => self.common_aux.extracted_values.clone(),- ArrayId::AttentionPartials => self.common_aux.attention_partials.clone(),- ArrayId::AttentionSums => self.common_aux.attention_sums.clone(),- ArrayId::AttentionMaxs => self.common_aux.attention_maxs.clone(),-- // Shared buffer arrays- ArrayId::RopeCosines(_) | ArrayId::RopeSines(_) => {- self.shared_buffer_array(id).expect("Shared buffer array should be available")- },-- // LLM-specific arrays- ArrayId::Logits => self.llm_state().logits.clone(),- ArrayId::TokenSeeds => self.llm_state().token_seeds.clone(),- ArrayId::Keys(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_transformer().expect("Expected transformer layer").keys.clone()- },- ArrayId::Values(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_transformer().expect("Expected transformer layer").values.clone()- },- ArrayId::AttentionSinks(layer_index) => {- self.shared_buffers.borrow().attention_sinks.as_ref().expect("Attention sinks not initialized")- [layer_index]- .clone()- },-- // SSM arrays (LLM only)- ArrayId::SsmInProj => {- self.llm_aux.as_ref().and_then(|aux| aux.ssm_inproj.clone()).expect("SSM inproj not initialized")- },- ArrayId::SsmPacked(_) => {- self.llm_aux.as_ref().and_then(|aux| aux.ssm_packed.clone()).expect("SSM packed not initialized")- },- ArrayId::SsmConvState(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_state_space().expect("Expected SSM layer").conv_state.clone()- },- ArrayId::SsmState(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_state_space().expect("Expected SSM layer").ssm_state.clone()- },- ArrayId::SsmX(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_x.clone()).expect("SSM x not initialized"),- ArrayId::SsmB(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_b.clone()).expect("SSM b not initialized"),- ArrayId::SsmC(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_c.clone()).expect("SSM c not initialized"),- ArrayId::SsmDt(_) => {- self.llm_aux.as_ref().and_then(|aux| aux.ssm_dt.clone()).expect("SSM dt not initialized")- },- ArrayId::SsmZ(_) => self.llm_aux.as_ref().and_then(|aux| aux.ssm_z.clone()).expect("SSM z not initialized"),- ArrayId::ShortConvState(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").conv_state.clone()- },- ArrayId::ShortConvSuffixState(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_short_conv().expect("Expected ShortConv layer").suffix_state.clone()- },- ArrayId::DeltaNetConvState(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").conv_state.clone()- },- ArrayId::DeltaNetSsmState(layer_index) => {- let cache = self.llm_state().cache_layers.borrow();- cache.data[layer_index].as_delta_net().expect("Expected DeltaNet layer").ssm_state.clone()- },-- // MoE arrays (LLM only)- ArrayId::MoeTopkIds => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_topk_ids.clone()).expect("MoE topk_ids not initialized")- },- ArrayId::MoeTopkProbs => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_topk_probs.clone())- .expect("MoE topk_probs not initialized"),- ArrayId::MoeOffsets => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_offsets.clone()).expect("MoE offsets not initialized")- },- ArrayId::MoeSumK => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_sumk.clone()).expect("MoE sumk not initialized")- },- ArrayId::MoeBucketedTokenIds => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_bucketed_token_ids.clone())- .expect("MoE bucketed_token_ids not initialized"),- ArrayId::MoeBucketedProbs => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_bucketed_probs.clone())- .expect("MoE bucketed_probs not initialized"),- ArrayId::MoeXPerm => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_x_perm.clone()).expect("MoE x_perm not initialized")- },- ArrayId::MoeTok2Row => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_tok2row.clone()).expect("MoE tok2row not initialized")- },- ArrayId::MoeYPartial => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_y_partial.clone()).expect("MoE y_partial not initialized")- },- ArrayId::MoeHidden => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_hidden.clone()).expect("MoE hidden not initialized")- },- ArrayId::MoeTwoPassRowExpertMap => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_two_pass_row_expert_map.clone())- .expect("MoE two_pass_row_expert_map not initialized"),- ArrayId::MoeTileCounts => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_tile_counts.clone())- .expect("MoE tile_counts not initialized"),- ArrayId::MoeTileOffsets => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_tile_offsets.clone())- .expect("MoE tile_offsets not initialized"),- ArrayId::MoeTileMap => {- self.llm_aux.as_ref().and_then(|aux| aux.moe_tile_map.clone()).expect("MoE tile_map not initialized")- },- ArrayId::MoeTotalTiles => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_total_tiles.clone())- .expect("MoE total_tiles not initialized"),- ArrayId::MoeDispatchArgs => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_dispatch_args.clone())- .expect("MoE dispatch_args not initialized"),- ArrayId::MoeScatterPartials => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_scatter_partials.clone())- .expect("MoE scatter_partials not initialized"),- ArrayId::MoeScatterBlockBases => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_scatter_block_bases.clone())- .expect("MoE scatter_block_bases not initialized"),- ArrayId::MoeBlockAlloc => self- .llm_aux- .as_ref()- .and_then(|aux| aux.moe_block_alloc.clone())- .expect("MoE block_alloc not initialized"),-- // Classifier-specific arrays- ArrayId::ClassifierPooling => self.classifier_state().pooling.clone(),- ArrayId::ClassifierPredictionHeadDense => self.classifier_state().dense.clone(),- ArrayId::ClassifierPredictionHeadNorm => self.classifier_state().norm.clone(),- ArrayId::ClassifierPredictionHeadLogits => self.classifier_state().classifier_logits.clone(),- }- }-- fn shared_buffer_array(- &self,- id: ArrayId,- ) -> Option<Array<B>> {- let shared = self.shared_buffers.borrow();- match id {- ArrayId::RopeCosines(rope_type) => Some(match rope_type {- RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").cosines.clone(),- RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").cosines.clone(),- }),- ArrayId::RopeSines(rope_type) => Some(match rope_type {- RopeType::Global => shared.global_rope.as_ref().expect("Global rope not initialized").sines.clone(),- RopeType::Local => shared.local_rope.as_ref().expect("Local rope not initialized").sines.clone(),- }),- _ => None,- }- }-- pub fn conv_padded_buffer(&self) -> Option<Array<B>> {- self.llm_aux.as_ref().and_then(|aux| aux.ssm_conv_padded.clone())- }-- // ========================================================================- // Public API Methods (formerly trait methods)- // ========================================================================-- pub fn active_row_count(&self) -> usize {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count,- ForwardPassMode::Classifier(state) => state.active_row_count,- }- }-- pub fn set_active_row_count(- &mut self,- active_row_count: usize,- ) {- match &mut self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.active_row_count = active_row_count,- ForwardPassMode::Classifier(state) => state.active_row_count = active_row_count,- }- }-- /// Start index (within the suffix batch) for which we need logits/sampling.- pub fn sampling_start(&self) -> usize {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_start,- ForwardPassMode::Classifier(_) => 0,- }- }-- /// Number of batch items for which we need logits/sampling.- pub fn sampling_length(&self) -> usize {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_length,- ForwardPassMode::Classifier(_) => self.common_aux.suffix_length,- }- }-- pub fn is_prefilling(&self) -> bool {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.is_prefilling,- ForwardPassMode::Classifier(_) => true,- }- }-- pub fn cache_layers(&self) -> Option<&Rc<RefCell<CacheLayers<B>>>> {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => Some(&state.cache_layers),- ForwardPassMode::Classifier(_) => None,- }- }-- pub fn sampling_output(&self) -> Option<&Array<B>> {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_output.as_ref(),- ForwardPassMode::Classifier(_) => None,- }- }-- #[cfg(feature = "tracing")]- pub fn traces(&self) -> &Rc<RefCell<ActivationTrace<B>>> {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => &state.traces,- ForwardPassMode::Classifier(state) => &state.traces,- }- }-- pub fn sampling_method_mut(&mut self) -> Option<&mut Option<SamplingMethod>> {- match &mut self.mode {- ForwardPassMode::LanguageModelGenerator(state) => Some(&mut state.sampling_method),- ForwardPassMode::Classifier(_) => None,- }- }-- pub fn sampling_method(&self) -> Option<SamplingMethod> {- match &self.mode {- ForwardPassMode::LanguageModelGenerator(state) => state.sampling_method,- ForwardPassMode::Classifier(_) => None,- }- }-- #[cfg(feature = "tracing")]- pub fn encode_copy_array(- &self,- encoder: &mut Encoder<B>,- source_array_id: ArrayId,- destination_array: Array<B>,- ) {- let source_array = self.array(source_array_id);-- let src_buf_rc = source_array.buffer();- let dst_buf_rc = destination_array.buffer();-- let copy_size_bytes = destination_array.size();- debug_assert_eq!(destination_array.size(), source_array.size());-- encoder.encode_copy(- src_buf_rc.borrow().deref(),- 0..copy_size_bytes,- dst_buf_rc.borrow_mut().deref_mut(),- 0..copy_size_bytes,- );- }-}diff --git a/crates/backend-uzu/src/forward_pass/token_inputs.rs b/crates/backend-uzu/src/forward_pass/token_inputs.rsnew file mode 100644index 000000000..2c07be9f6--- /dev/null+++ b/crates/backend-uzu/src/forward_pass/token_inputs.rs@@ -0,0 +1,174 @@+use ndarray::ArrayView2;++#[cfg(feature = "tracing")]+use crate::forward_pass::traces::ActivationTrace;+use crate::{+ DataType,+ array::{Array, ArrayContextExt},+ backends::common::{Allocation, Backend},+ encodable_block::DecoderArguments,+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},+};++pub struct TokenInputs<B: Backend> {+ token_ids: Array<B>,+ token_subtrie_ranges: Option<Array<B>>,+ token_positions: Array<B>,+ token_parents: Array<B>,+}++impl<B: Backend> TokenInputs<B> {+ pub fn new_llm(+ context: &B::Context,+ model_shape: &ModelShape,+ token_ids: &[u64],+ token_subtrie_ranges: Option<&[[u32; 3]]>,+ token_positions: &[usize],+ token_ids_array: Option<Array<B>>,+ token_positions_array: Option<Array<B>>,+ sampling_start: usize,+ sampling_length: usize,+ ) -> Self {+ let suffix_length = token_ids.len();+ assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");++ Self {+ token_ids: token_ids_array.unwrap_or_else(|| Self::init_token_ids(context, token_ids)),+ token_subtrie_ranges: token_subtrie_ranges+ .map(|ranges| Self::init_token_subtrie_ranges(context, model_shape, suffix_length, ranges)),+ token_positions: token_positions_array+ .unwrap_or_else(|| Self::init_token_positions(context, token_positions)),+ token_parents: Self::init_token_parents(context, token_positions, sampling_start, sampling_length),+ }+ }++ pub fn new_classifier(+ context: &B::Context,+ token_ids: &[u64],+ token_positions: &[usize],+ ) -> Self {+ let suffix_length = token_ids.len();+ assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");++ Self {+ token_ids: Self::init_token_ids(context, token_ids),+ token_subtrie_ranges: None,+ token_positions: Self::init_token_positions(context, token_positions),+ token_parents: Self::init_token_parents(context, token_positions, 0, 0),+ }+ }++ pub fn token_ids(&self) -> &Allocation<B> {+ self.token_ids.allocation()+ }++ pub fn token_positions(&self) -> &Allocation<B> {+ self.token_positions.allocation()+ }++ pub fn token_parents(&self) -> &Allocation<B> {+ self.token_parents.allocation()+ }++ pub fn token_subtrie_ranges(&self) -> Option<&Allocation<B>> {+ self.token_subtrie_ranges.as_ref().map(Array::allocation)+ }++ pub fn decoder_arguments<'a>(+ &'a self,+ model_shape: &ModelShape,+ shared_buffers: &'a SharedBuffers<B>,+ cache_layers: Option<&'a mut CacheLayers<B>>,+ batch_dim: usize,+ sampling_start: usize,+ sampling_length: usize,+ #[cfg(feature = "tracing")] trace: Option<&'a mut ActivationTrace<B>>,+ ) -> DecoderArguments<'a, B> {+ DecoderArguments {+ token_positions: self.token_positions(),+ token_parents: self.token_parents(),+ token_subtrie_ranges: self.token_subtrie_ranges(),+ shared_buffers,+ cache_layers,+ batch_dim,+ sampling_start,+ sampling_length,+ rope_max_sequence_length: model_shape.context_length(),+ rope_dim: model_shape.rope_dim(),+ #[cfg(feature = "tracing")]+ trace,+ }+ }++ fn init_token_ids(+ context: &B::Context,+ token_ids: &[u64],+ ) -> Array<B> {+ context.create_array_from(&[token_ids.len()], token_ids)+ }++ fn init_token_positions(+ context: &B::Context,+ token_positions: &[usize],+ ) -> Array<B> {+ let positions_i32: Box<[i32]> = token_positions.iter().map(|position| *position as i32).collect();+ context.create_array_from(&[token_positions.len()], positions_i32.as_ref())+ }++ fn init_token_parents(+ context: &B::Context,+ token_positions: &[usize],+ sampling_start: usize,+ sampling_length: usize,+ ) -> Array<B> {+ let suffix_length = token_positions.len();+ let mut parents = vec![-1i32; suffix_length];++ if sampling_length > 0 {+ let root_pos = token_positions.get(sampling_start).copied().unwrap_or(0);+ let mut stack: Vec<usize> = Vec::new();++ for local_idx in 0..sampling_length {+ let abs_idx = sampling_start + local_idx;+ let Some(&pos) = token_positions.get(abs_idx) else {+ break;+ };+ let depth = pos.saturating_sub(root_pos);+ let parent_local = if depth == 0 {+ -1+ } else if let Some(&parent) = stack.get(depth - 1) {+ parent as i32+ } else {+ debug_assert!(false, "invalid trie depth ordering: depth={depth}, stack_len={}", stack.len());+ -1+ };+ parents[abs_idx] = parent_local;++ if stack.len() <= depth {+ stack.resize(depth + 1, 0);+ }+ stack[depth] = local_idx;+ stack.truncate(depth + 1);+ }+ }++ context.create_array_from(&[suffix_length], &parents)+ }++ fn init_token_subtrie_ranges(+ context: &B::Context,+ model_shape: &ModelShape,+ suffix_length: usize,+ token_subtrie_ranges: &[[u32; 3]],+ ) -> Array<B> {+ let shape = model_shape.subtrie_ranges_shape(suffix_length);+ let mut array = context.create_array_zeros(&shape, DataType::U32);+ let source = ArrayView2::from_shape(+ (token_subtrie_ranges.len(), 3),+ bytemuck::cast_slice::<_, u32>(token_subtrie_ranges),+ )+ .expect("Invalid token subtrie range shape");+ array.copy_from_view(source);+ array+ }+}diff --git a/crates/backend-uzu/src/forward_pass/traces.rs b/crates/backend-uzu/src/forward_pass/traces.rsindex 37b4cb589..820073ff1 100644--- a/crates/backend-uzu/src/forward_pass/traces.rs+++ b/crates/backend-uzu/src/forward_pass/traces.rs@@ -1,29 +1,15 @@-use std::{cell::RefCell, rc::Rc};-use crate::{- DataType,array::{Array, ArrayContextExt},backends::common::Backend,forward_pass::model_shape::ModelShape,};-fn create_trace_array<B: Backend>(- context: &B::Context,- shape: &[usize],- data_type: DataType,- label: &str,-) -> Array<B> {- context.create_array_uninitialized(shape, data_type, label)-}-fn create_layer_results<B: Backend>(context: &B::Context,model_shape: &ModelShape,suffix_length: usize,-) -> Vec<Rc<RefCell<LayerActivationTrace<B>>>> {- (0..model_shape.num_layers)- .map(|_| Rc::new(RefCell::new(LayerActivationTrace::new(context, model_shape, suffix_length))))- .collect()+) -> Box<[LayerActivationTrace<B>]> {+ (0..model_shape.num_layers).map(|_| LayerActivationTrace::new(context, model_shape, suffix_length)).collect()}pub struct LayerActivationTrace<B: Backend> {@@ -46,25 +32,25 @@ impl<B: Backend> LayerActivationTrace<B> {) -> Self {let main_shape = model_shape.main_shape(suffix_length);let activation_data_type = model_shape.activation_data_type();- let main = |label| create_trace_array(context, &main_shape, activation_data_type, label);+ let main = || context.create_array_uninitialized(&main_shape, activation_data_type);Self {- inputs: main("layer_activation_trace_inputs"),- pre_attention_norm: main("layer_activation_trace_pre_attention_norm"),- attention: main("layer_activation_trace_attention"),- post_attention_norm: main("layer_activation_trace_post_attention_norm"),- mlp_inputs: main("layer_activation_trace_mlp_inputs"),- pre_mlp_norm: main("layer_activation_trace_pre_mlp_norm"),- mlp: main("layer_activation_trace_mlp"),- post_mlp_norm: main("layer_activation_trace_post_mlp_norm"),- outputs: main("layer_activation_trace_outputs"),+ inputs: main(),+ pre_attention_norm: main(),+ attention: main(),+ post_attention_norm: main(),+ mlp_inputs: main(),+ pre_mlp_norm: main(),+ mlp: main(),+ post_mlp_norm: main(),+ outputs: main(),}}}pub struct ActivationTrace<B: Backend> {pub embedding_norm: Option<Array<B>>,- pub layer_results: Vec<Rc<RefCell<LayerActivationTrace<B>>>>,+ pub layer_results: Box<[LayerActivationTrace<B>]>,pub output_norm: Array<B>,pub output_pooling: Option<Array<B>>,pub logits: Array<B>,@@ -83,14 +69,9 @@ impl<B: Backend> ActivationTrace<B> {Self {embedding_norm: None,layer_results,- output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),+ output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),output_pooling: None,- logits: create_trace_array(- context,- &model_shape.logits_shape(suffix_length),- activation_data_type,- "activation_trace_logits",- ),+ logits: context.create_array_uninitialized(&model_shape.logits_shape(suffix_length), activation_data_type),}}@@ -106,29 +87,19 @@ impl<B: Backend> ActivationTrace<B> {let model_dim = model_shape.main_shape(1)[1];Self {- embedding_norm: Some(create_trace_array(- context,- &main_shape,- activation_data_type,- "activation_trace_embedding_norm",- )),+ embedding_norm: Some(context.create_array_uninitialized(&main_shape, activation_data_type)),layer_results,- output_norm: create_trace_array(context, &main_shape, activation_data_type, "activation_trace_output_norm"),- output_pooling: Some(create_trace_array(- context,- &[1, model_dim],- activation_data_type,- "activation_trace_output_pooling",- )),- logits: create_trace_array(context, &[1, num_labels], activation_data_type, "activation_trace_logits"),+ output_norm: context.create_array_uninitialized(&main_shape, activation_data_type),+ output_pooling: Some(context.create_array_uninitialized(&[1, model_dim], activation_data_type)),+ logits: context.create_array_uninitialized(&[1, num_labels], activation_data_type),}}- pub fn embedding_norm(&self) -> &Array<B> {- self.embedding_norm.as_ref().expect("embedding_norm is only available for classifier traces")+ pub fn embedding_norm_mut(&mut self) -> &mut Array<B> {+ self.embedding_norm.as_mut().expect("embedding_norm is only available for classifier traces")}- pub fn output_pooling(&self) -> &Array<B> {- self.output_pooling.as_ref().expect("output_pooling is only available for classifier traces")+ pub fn output_pooling_mut(&mut self) -> &mut Array<B> {+ self.output_pooling.as_mut().expect("output_pooling is only available for classifier traces")}}diff --git a/crates/backend-uzu/src/language_model/grammar/mod.rs b/crates/backend-uzu/src/language_model/grammar/mod.rsindex 43e9e01c3..bf464d854 100644--- a/crates/backend-uzu/src/language_model/grammar/mod.rs+++ b/crates/backend-uzu/src/language_model/grammar/mod.rs@@ -20,10 +20,11 @@ pub trait CompiledGrammar {fn is_terminated(&self) -> bool;}+#[cfg_attr(not(grammar_xgrammar), allow(unused_variables))]pub fn create_compiled_grammar(- #[allow(unused)] config: &GrammarConfig,- #[allow(unused)] tokenizer: &Tokenizer,- #[allow(unused)] stop_token_ids: Option<&[i32]>,+ config: &GrammarConfig,+ tokenizer: &Tokenizer,+ stop_token_ids: Option<&[i32]>,) -> Result<Box<dyn CompiledGrammar>, Error> {#[cfg(grammar_xgrammar)]{@@ -38,5 +39,7 @@ pub fn create_compiled_grammar(}#[cfg(not(grammar_xgrammar))]- Err(Error::GrammarNoBackendAvailable)+ {+ Err(Error::GrammarNoBackendAvailable)+ }}diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rsindex a0b48f61c..79c13d59b 100644--- a/crates/backend-uzu/src/language_model/language_model_generator.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs@@ -1,10 +1,4 @@-use std::{- any::Any,- iter::repeat_n,- ops::{Deref, DerefMut, Range},- path::Path,- time::Instant,-};+use std::{any::Any, iter::repeat_n, ops::Range, path::Path, time::Instant};use itertools::{Either, Itertools, izip};@@ -15,17 +9,14 @@ use super::{rng::PRng,};use crate::{+ array::{Array, ArrayContextExt},backends::common::{- Backend, CommandBuffer, Context, DenseBuffer, Encoder, Executable,- kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},+ Allocation, AsBufferRangeRef, Backend, CommandBuffer, Context, DenseBuffer, Encoder, Pending,+ kernel::TokenCopySampledKernel,},config::ModelMetadata,- encodable_block::EncodingParameters,- forward_pass::{- cache_layers::{CacheLayer, CacheLayersSlice},- kv_cache_layer::INVALID_POSITION,- state::ForwardPassState,- },+ encodable_block::{DecoderDecodeInput, EncodingParameters, SamplingArguments, SamplingInputs},+ forward_pass::{cache_layers::CacheLayersSlice, kv_cache_layer::INVALID_POSITION, token_inputs::TokenInputs},language_model::grammar::CompiledGrammar,session::{config::DecodingConfig,@@ -44,24 +35,22 @@ struct Task<'a> {token_positions: &'a [usize],token_bitmask: Option<&'a [u32]>,token_seeds: &'a [u64],- expected_number_of_new_tokens: usize,active_row_count: usize,sampling_start: usize,sampling_length: usize,is_prefilling: bool,}-#[derive(Debug, Clone, PartialEq)]-struct TaskEncodingKey {- context_len: usize,- batch_size: usize,- expected_number_of_new_tokens: usize,- active_row_count: usize,- sampling_method: SamplingMethod,- sampling_start: usize,- sampling_len: usize,- has_bitmask: bool,- is_prefilling: bool,+struct ForwardPassResources<B: Backend> {+ token_inputs: TokenInputs<B>,+ logits: Option<Allocation<B>>,+ sampling_output: Option<Allocation<B>>,+ sampling_inputs: Option<SamplingInputs<B>>,+}++struct InFlightForwardPass<B: Backend> {+ resources: ForwardPassResources<B>,+ pending: Pending<B>,}pub struct LanguageModelGenerator<B: Backend> {@@ -69,7 +58,8 @@ pub struct LanguageModelGenerator<B: Backend> {pub tokens: Vec<u64>,pub context: LanguageModelGeneratorContext<B>,- pre_encoded_task: Option<(TaskEncodingKey, Executable<B>)>,+ async_in_flight: Vec<Option<InFlightForwardPass<B>>>,+ async_token_ids: Option<Array<B>>,registered_prefix_len: usize,gpu_capture: GpuCaptureManager<B>,}@@ -100,6 +90,10 @@ pub trait LanguageModelGeneratorTrait {sampling_method: SamplingMethod,on_complete: Box<dyn FnOnce(u64) + Send>,) -> Result<(), Error>;+ fn finish_async(+ &mut self,+ pass_idx: usize,+ );fn clear_cache(&mut self);fn reset_state(&mut self);@@ -187,7 +181,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let token_seeds = repeat_n(0, tokens_length - 1).chain(flat_trie.token_seeds()).chunks(prefill_step_size);- let mut last_state: Option<ForwardPassState<B>> = None;+ let mut last_sampling_output: Option<Allocation<B>> = None;+ let mut last_sampling_length = 0;let mut run_times: Vec<f64> = Vec::new();// Process each prefill step and update the KV cache.@@ -269,7 +264,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let _ = self.gpu_capture.start_capture(&self.context.context, "prefill");}- let _ = last_state.take();+ let _ = last_sampling_output.take();let task = Task {token_ids: &step_token_ids,@@ -277,14 +272,13 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {token_positions: &step_token_positions,token_bitmask: step_token_bitmask.as_deref(),token_seeds: &step_token_seeds,- expected_number_of_new_tokens: step_token_ids.len(),active_row_count,sampling_start,sampling_length,is_prefilling: !should_sample_after_step,};- let (state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;+ let (sampling_output, run_time) = self.run_model(task, sampling_method)?;if should_capture {self.gpu_capture.stop_capture(&self.context.context, "prefill").map_err(|_| Error::CaptureFailed)?;@@ -306,11 +300,12 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.registered_prefix_len = prefix_offset + tokens_start_index + tokens_processed_this_step;}- last_state = Some(state);+ last_sampling_length = sampling_length;+ last_sampling_output = sampling_output;run_times.push(run_time);}- let mut final_state = last_state.ok_or(Error::PrefillFailed)?;+ let final_sampling_output = last_sampling_output;if !sample_suffix {self.sync_prefix();return Ok(PrefillResult {@@ -318,7 +313,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {forwardpass_durations: run_times,});}- let sampled_tokens = self.read_sampling_output(&mut final_state)?;+ let sampled_tokens = self.read_sampling_output(+ final_sampling_output.as_ref().expect("sampling output must exist"),+ last_sampling_length,+ )?;let last_suffix_start = prefill_step_size * (prefill_steps - 1);let suffix_root_index = (tokens_length - last_suffix_start) - 1;@@ -403,16 +401,16 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {token_positions: &token_positions,token_bitmask: token_bitmask.as_deref(),token_seeds: &token_seeds,- expected_number_of_new_tokens: 1,active_row_count,sampling_start: 0,sampling_length: active_row_count,is_prefilling: false,};- let (mut state, run_time) = self.run_model(task, self.allow_pre_encode(), sampling_method)?;-- let sampled_tokens = self.read_sampling_output(&mut state)?;+ let sampling_length = task.sampling_length;+ let (sampling_output, run_time) = self.run_model(task, sampling_method)?;+ let sampled_tokens =+ self.read_sampling_output(sampling_output.as_ref().expect("sampling output must exist"), sampling_length)?;let (accepted_tokens, accepted_token_indices) =flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());@@ -443,6 +441,8 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.async_buffers.prepare_positions(prefill_count, tokens_to_generate);self.context.async_buffers.prepare_seeds(&self.context.seed, prefill_count, tokens_to_generate);self.context.async_buffers.reset_counter();+ self.async_in_flight = (0..self.context.async_buffers.batch_size).map(|_| None).collect();+ self.async_token_ids = None;}/// Submits a single async forward pass.@@ -465,16 +465,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let batch_size = self.context.async_buffers.batch_size;let slot = pass_idx % batch_size;- let results_buffer = self.context.async_buffers.results.clone();- let async_positions_buffer = self.context.async_buffers.positions.clone();- let async_seeds_buffer = self.context.async_buffers.seeds.clone();-let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;-- let token_position = unsafe {- let ptr = async_positions_buffer.borrow().cpu_ptr().as_ptr() as *const u32;- *ptr.add(pass_idx) as usize- };+ let token_position = self.context.async_buffers.positions.as_slice::<i32>()[pass_idx] as usize;+ let token_seed = self.context.async_buffers.seeds.as_slice::<u64>()[pass_idx];let task = Task {token_ids: &[last_token],@@ -482,17 +475,18 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {token_positions: &[token_position],token_bitmask: None,token_seeds: &[0], // Ignored, using async buffer- expected_number_of_new_tokens: 1,active_row_count: 1,sampling_start: 0,sampling_length: 1,is_prefilling: false,};- let async_positions = Some((async_positions_buffer.clone(), pass_idx));- let async_seeds = Some((async_seeds_buffer.clone(), pass_idx));-- let skip_token_ids_copy = pass_idx > 0;+ let token_ids_array = if pass_idx > 0 {+ Some(self.async_token_ids.take().expect("previous async pass must provide token_ids array"))+ } else {+ None+ };+ let sampling_inputs = SamplingInputs::from_slices(self.context.context.as_ref(), &[token_seed], None, None);let is_first_decode = !is_continuation;let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);@@ -500,65 +494,47 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let _ = self.gpu_capture.start_capture(&self.context.context, "decode");}- let mut state = ForwardPassState::new_llm(- self.context.context.clone(),- &self.context.decoder_config,- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.cache_layers.clone(),- self.context.shared_buffers.clone(),- &task.token_ids,- task.token_subtrie_ranges,- &task.token_positions,- task.token_bitmask,- &task.token_seeds,- task.active_row_count,- /*sampling_start=*/ 0,- /*sampling_length=*/ task.active_row_count,- task.is_prefilling,- skip_token_ids_copy,- async_positions,- async_seeds,- );- if let Some(sm) = state.sampling_method_mut() {- *sm = Some(sampling_method);- }-+ let batch_dim = task.active_row_count;+ let sampling_start = task.sampling_start;+ let sampling_length = task.sampling_length;+ let token_inputs = self.build_token_inputs(task, token_ids_array, None);let mut encoder = Encoder::<B>::new(self.context.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;-- // Wait on previous pass if this is a continuationif is_continuation {encoder.encode_wait_for_event(&self.context.async_buffers.event, current_counter);}-- self.context- .executables- .encode(&mut state, &EncodingParameters::new(), &mut encoder)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- // Encode sampling- self.context.gpu_sampler.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ let resources = Self::encode_forward_pass_on(+ &self.context,+ &mut encoder,+ token_inputs,+ batch_dim,+ sampling_start,+ sampling_length,+ false,+ Some(sampling_method),+ &EncodingParameters::new(),+ Some(sampling_inputs),+ )?;// Copy sampled token: sampling_output → token_ids (for next pass)- // and sampling_output → results[slot] (for callback)- let sampling_output = state.sampling_output().expect("sampling_output must exist after sampling encode");- let sampling_output_buf_rc = sampling_output.buffer();- let sampling_output_buf_borrow = sampling_output_buf_rc.borrow();- let token_ids_buf_rc = self.context.scratch_buffers.token_ids.buffer();- let mut token_ids_buf_borrow = token_ids_buf_rc.borrow_mut();-+ let token_ids_shape = [1];+ let token_ids_data_type = crate::DataType::U64;+ let mut async_token_ids_allocation =+ self.context.context.create_array_uninitialized(&token_ids_shape, token_ids_data_type).into_allocation();+ let async_token_ids_buffer_range = async_token_ids_allocation.as_buffer_range_ref();+ let async_token_ids_range = async_token_ids_buffer_range.range();+ let async_token_ptr = SendPtr(unsafe {+ (async_token_ids_buffer_range.buffer().cpu_ptr().as_ptr() as *const u64)+ .add(async_token_ids_range.start / std::mem::size_of::<u64>())+ });self.context.token_copy_sampled.encode(- sampling_output_buf_borrow.deref(),- token_ids_buf_borrow.deref_mut(),- &mut encoder,- );- let results_offset = slot * std::mem::size_of::<u32>();- self.context.token_copy_results.encode(- sampling_output_buf_borrow.deref(),- (results_buffer.borrow_mut().deref_mut(), results_offset),+ resources.sampling_output.as_ref().expect("Sampling output must exist"),+ &mut async_token_ids_allocation,&mut encoder,);+ self.async_token_ids = Some(unsafe {+ Array::from_allocation(async_token_ids_allocation, 0, &token_ids_shape, token_ids_data_type)+ });// Scatter + register for all transformer layersself.context.cache_layers.borrow_mut().update_after_acceptance(@@ -575,11 +551,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.async_buffers.counter.set(next_counter);// Add completion handler- let results_buffer_ptr = SendPtr(results_buffer.borrow().cpu_ptr().as_ptr() as *const u32);-let handler = move |result: Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>| {result.expect("async decoding forward pass completed with error");- let token = { unsafe { *results_buffer_ptr.as_ptr().add(slot) as u64 } };+ let token = unsafe { *async_token_ptr.as_ptr() };on_complete(token);};@@ -588,22 +562,43 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let pending = encoder.end_encoding().submit();if should_capture {- pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ let completed = pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ drop(resources);+ drop(completed);self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;+ } else {+ assert!(self.async_in_flight[slot].is_none(), "async slot {slot} still holds an in-flight state");+ self.async_in_flight[slot] = Some(InFlightForwardPass {+ resources,+ pending,+ });}Ok(())}- fn clear_cache(&mut self) {- self.pre_encoded_task = None;+ fn finish_async(+ &mut self,+ pass_idx: usize,+ ) {+ let slot = pass_idx % self.context.async_buffers.batch_size;+ if let Some(in_flight) = self.async_in_flight[slot].take() {+ let InFlightForwardPass {+ resources,+ pending,+ } = in_flight;+ drop((resources, pending));+ }}+ fn clear_cache(&mut self) {}+fn reset_state(&mut self) {- self.context.cache_layers.borrow_mut().clear();+ self.context.cache_layers.borrow_mut().clear(self.context.context.as_ref());self.tokens.clear();self.registered_prefix_len = 0;- self.pre_encoded_task = None;+ self.async_in_flight.clear();+ self.async_token_ids = None;self.gpu_capture.reset();let seed = self.decoding_config.sampling_seed.resolve();@@ -659,29 +654,9 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {context: &dyn Any,) {let ctx = context.downcast_ref::<LlmContext<B>>().unwrap();- let mut llm_state = self.context.cache_layers.borrow_mut();- for (_layer_idx, (ctx_layer, gen_layer)) in- ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()- {- match (ctx_layer, gen_layer) {- (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {- let copy_rows = src.prefix_segment_length();- if copy_rows > 0 {- dst.keys.copy_slice(&src.keys, 0, 0..copy_rows, 0);- dst.values.copy_slice(&src.values, 0, 0..copy_rows, 0);- }- dst.state = src.state.clone();- },- (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {- dst.conv_state.copy_from_array(&src.conv_state);- dst.ssm_state.copy_from_array(&src.ssm_state);- },- _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),- }- }- drop(llm_state);-+ self.context.cache_layers.borrow_mut().copy_from(&ctx.cache_layers, self.context.context.as_ref());self.tokens = ctx.tokens.clone();+ self.registered_prefix_len = self.tokens.len().saturating_sub(1);}}@@ -699,7 +674,8 @@ impl<B: Backend> LanguageModelGenerator<B> {decoding_config,tokens: Vec::new(),context,- pre_encoded_task: None,+ async_in_flight: Vec::new(),+ async_token_ids: None,registered_prefix_len: 0,gpu_capture,})@@ -708,80 +684,52 @@ impl<B: Backend> LanguageModelGenerator<B> {fn run_model(&mut self,task: Task,- allow_pre_encode: bool,sampling_method: SamplingMethod,- ) -> Result<(ForwardPassState<B>, f64), Error> {+ ) -> Result<(Option<Allocation<B>>, f64), Error> {let run_start = Instant::now();-- let mut state = ForwardPassState::new_llm(- self.context.context.clone(),- &self.context.decoder_config,- &self.context.model_shape,- &self.context.scratch_buffers,- self.context.cache_layers.clone(),- self.context.shared_buffers.clone(),- task.token_ids,- task.token_subtrie_ranges,- task.token_positions,- task.token_bitmask,- task.token_seeds,- task.active_row_count,- task.sampling_start,- task.sampling_length,- task.is_prefilling,- false,- None,- None,- );-- if let Some(method) = state.sampling_method_mut() {- *method = Some(sampling_method);- }-- let encoding_key = TaskEncodingKey {- context_len: self.tokens.len(),- batch_size: task.token_ids.len(),- expected_number_of_new_tokens: task.expected_number_of_new_tokens,- active_row_count: task.active_row_count,- sampling_method,- sampling_start: task.sampling_start,- sampling_len: task.sampling_length,- has_bitmask: task.token_bitmask.is_some(),- is_prefilling: task.is_prefilling,- };+ let sample = !task.is_prefilling;+ let is_prefilling = task.is_prefilling;+ let batch_dim = task.active_row_count;+ let sampling_start = task.sampling_start;+ let sampling_length = task.sampling_length;+ let sampling_inputs = sample.then(|| {+ self.create_sampling_inputs(task.sampling_start, task.sampling_length, task.token_seeds, task.token_bitmask)+ });let is_first_decode = task.token_ids.len() == 1;let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);+ let token_inputs = self.build_token_inputs(task, None, None);if should_capture {self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;- self.pre_encoded_task = None;}- let sample = !task.is_prefilling;-- let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()- && pre_encoded_key == encoding_key- {- pre_encoded_executable- } else {- self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?- };-- let pending = executable.submit();-- if allow_pre_encode {- let mut next_encoding_key = encoding_key;-- next_encoding_key.context_len += 1;-- let next_executable =- self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;+ let resources = Self::encode_forward_pass_on(+ &self.context,+ &mut encoder,+ token_inputs,+ batch_dim,+ sampling_start,+ sampling_length,+ is_prefilling,+ sample.then_some(sampling_method),+ &EncodingParameters::new(),+ sampling_inputs,+ )?;- self.pre_encoded_task = Some((next_encoding_key, next_executable));- }+ let pending = encoder.end_encoding().submit();- pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ let completed = pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ let ForwardPassResources {+ token_inputs,+ logits,+ sampling_output,+ sampling_inputs,+ } = resources;+ drop((token_inputs, logits, sampling_inputs));+ drop(completed);let run_time = run_start.elapsed().as_secs_f64();@@ -789,49 +737,140 @@ impl<B: Backend> LanguageModelGenerator<B> {self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;}- Ok((state, run_time))+ Ok((sampling_output, run_time))}- fn encode_forward_pass(- &self,- state: &mut ForwardPassState<B>,+ fn encode_forward_pass_on(+ context: &LanguageModelGeneratorContext<B>,+ encoder: &mut Encoder<B>,+ token_inputs: TokenInputs<B>,+ batch_dim: usize,+ sampling_start: usize,+ sampling_length: usize,+ is_prefilling: bool,+ sampling_method: Option<SamplingMethod>,parameters: &EncodingParameters,- sample: bool,- ) -> Result<Executable<B>, Error> {- let mut encoder = Encoder::<B>::new(self.context.context.as_ref())- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;-- self.context- .executables- .encode(state, parameters, &mut encoder)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- if sample {- self.context.gpu_sampler.encode(state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ sampling_inputs: Option<SamplingInputs<B>>,+ ) -> Result<ForwardPassResources<B>, Error> {+ let mut sampling_output = sampling_method.map(|_| {+ context.context.create_array_uninitialized(&[sampling_length], crate::DataType::U32).into_allocation()+ });+ let mut logits = None;+ if is_prefilling {+ let mut cache_layers = context.cache_layers.borrow_mut();+ let decoder_arguments = token_inputs.decoder_arguments(+ &context.model_shape,+ context.shared_buffers.as_ref(),+ Some(&mut *cache_layers),+ batch_dim,+ sampling_start,+ sampling_length,+ #[cfg(feature = "tracing")]+ None,+ );+ context+ .executables+ .encode_prefill(decoder_arguments, token_inputs.token_ids(), parameters, encoder)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ } else {+ let mut cache_layers = context.cache_layers.borrow_mut();+ let decoder_arguments = token_inputs.decoder_arguments(+ &context.model_shape,+ context.shared_buffers.as_ref(),+ Some(&mut *cache_layers),+ batch_dim,+ sampling_start,+ sampling_length,+ #[cfg(feature = "tracing")]+ None,+ );+ let mut retained_logits = context+ .executables+ .encode_decode(+ decoder_arguments,+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),+ None,+ parameters,+ encoder,+ )+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;++ let sampling_inputs = sampling_inputs.as_ref().expect("Sampling requires sampling inputs");+ let sampling_result = context.gpu_sampler.encode(+ SamplingArguments {+ logits: &mut retained_logits,+ seeds: sampling_inputs.seeds.allocation(),+ bitmask: sampling_inputs.bitmask.as_ref().map(Array::allocation),+ output: sampling_output.as_mut().expect("Sampling requires output allocation"),+ sampling_method: sampling_method.expect("Sampling requires method"),+ batch_size: sampling_length,+ vocab_size: context.model_config.model_config.vocab_size,+ },+ encoder,+ );+ sampling_result.map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ logits = Some(retained_logits);}- let executable = encoder.end_encoding();+ Ok(ForwardPassResources {+ token_inputs,+ logits,+ sampling_output,+ sampling_inputs,+ })+ }++ fn build_token_inputs(+ &self,+ task: Task<'_>,+ token_ids_array: Option<Array<B>>,+ token_positions_array: Option<Array<B>>,+ ) -> TokenInputs<B> {+ TokenInputs::new_llm(+ self.context.context.as_ref(),+ &self.context.model_shape,+ task.token_ids,+ task.token_subtrie_ranges,+ task.token_positions,+ token_ids_array,+ token_positions_array,+ task.sampling_start,+ task.sampling_length,+ )+ }- Ok(executable)+ fn create_sampling_inputs(+ &self,+ sampling_start: usize,+ sampling_length: usize,+ token_seeds: &[u64],+ token_bitmask: Option<&[u32]>,+ ) -> SamplingInputs<B> {+ let seeds = &token_seeds[sampling_start..sampling_start + sampling_length];+ let bitmask_row_len = token_bitmask.map(|_| self.context.model_shape.bitmask_shape(1)[1]);+ let bitmask = token_bitmask.map(|mask| {+ let row_len = bitmask_row_len.expect("bitmask row length must exist");+ let start = sampling_start * row_len;+ let end = start + sampling_length * row_len;+ &mask[start..end]+ });+ SamplingInputs::from_slices(self.context.context.as_ref(), seeds, bitmask, bitmask_row_len)}fn read_sampling_output(- &mut self,- state: &mut ForwardPassState<B>,+ &self,+ sampling_output: &Allocation<B>,+ batch_size: usize,) -> Result<Vec<u64>, Error> {- let sampling_output = state- .sampling_output()- .expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");-- let output_view = sampling_output.as_view::<u32>();- let batch_size = state.sampling_length();-- let mut result = Vec::with_capacity(batch_size);- for i in 0..batch_size {- result.push(output_view[[i]] as u64);- }-- Ok(result)+ let buffer_range = sampling_output.as_buffer_range_ref();+ let range = buffer_range.range();+ let values = unsafe {+ std::slice::from_raw_parts(+ (buffer_range.buffer().cpu_ptr().as_ptr() as *const u32).add(range.start / std::mem::size_of::<u32>()),+ batch_size,+ )+ };+ Ok(values.iter().map(|value| *value as u64).collect())}fn update_cache_layers(@@ -861,14 +900,6 @@ impl<B: Backend> LanguageModelGenerator<B> {Ok(())}- fn allow_pre_encode(&self) -> bool {- let debug_active = self.context.context.debug_active();-- let result = self.decoding_config.allow_pre_encode && !debug_active;-- result- }-fn sync_prefix(&mut self) {if self.tokens.is_empty() {return;diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rsindex 12476d069..a32b45032 100644--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs@@ -7,15 +7,14 @@ use std::{use crate::{DataType,+ array::{Array, ArrayContextExt},backends::common::{- Backend, Context, DenseBuffer, Kernels,- kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},+ Backend, Context, Kernels,+ kernel::{TokenCopySampledKernel, kv_cache_update::KVCacheUpdate},},- config::{DecoderConfig, LanguageModelConfig, ModelMetadata},+ config::{LanguageModelConfig, ModelMetadata},encodable_block::{Decoder, Sampling},- forward_pass::{- cache_layers::CacheLayers, model_shape::ModelShape, scratch_buffers::ScratchBuffers, state::SharedBuffers,- },+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},language_model::rng::PRng,parameters::ParameterLoader,session::{@@ -25,18 +24,15 @@ use crate::{},};-/// Pre-allocated buffers for async generation pipeline.+/// Pre-allocated state for async generation pipeline./// Indexed by pass_idx to avoid race conditions between GPU passes.pub struct AsyncBuffers<B: Backend> {- /// Positions buffer: [max_tokens] i32+ /// Positions array: [max_tokens] i32/// Pre-populated with [prefill_count, prefill_count+1, ...]- pub positions: Rc<RefCell<B::DenseBuffer>>,- /// Seeds buffer: [max_tokens] u64+ pub positions: Array<B>,+ /// Seeds array: [max_tokens] u64/// Pre-populated with deterministic seed sequence- pub seeds: Rc<RefCell<B::DenseBuffer>>,- /// Results buffer: [batch_size] u32- /// Each pass writes its sampled token to results[pass_idx % batch_size]- pub results: Rc<RefCell<B::DenseBuffer>>,+ pub seeds: Array<B>,/// Event for GPU-side synchronization between passespub event: B::Event,/// Current event counter (pass N waits on N, signals N+1)@@ -53,18 +49,13 @@ impl<B: Backend> AsyncBuffers<B> {max_tokens: usize,batch_size: usize,) -> Self {- let positions =- context.create_buffer(max_tokens * std::mem::size_of::<i32>()).expect("Failed to create positions buffer");- let seeds =- context.create_buffer(max_tokens * std::mem::size_of::<u64>()).expect("Failed to create seeds buffer");- let results =- context.create_buffer(batch_size * std::mem::size_of::<u32>()).expect("Failed to create results buffer");+ let positions = context.create_array_uninitialized(&[max_tokens], DataType::I32);+ let seeds = context.create_array_uninitialized(&[max_tokens], DataType::U64);let event = context.create_event().expect("Failed to create event");Self {- positions: Rc::new(RefCell::new(positions)),- seeds: Rc::new(RefCell::new(seeds)),- results: Rc::new(RefCell::new(results)),+ positions,+ seeds,event,counter: Cell::new(0),prefill_count: Cell::new(0),@@ -75,32 +66,28 @@ impl<B: Backend> AsyncBuffers<B> {/// Prepare positions buffer: [prefill_count-1, prefill_count, ...]/// Uses prefill_count-1 to match sync path which uses self.tokens.len()-1pub fn prepare_positions(- &self,+ &mut self,prefill_count: usize,tokens_to_generate: usize,) {self.prefill_count.set(prefill_count);let base_position = prefill_count.saturating_sub(1);- let ptr = self.positions.borrow().cpu_ptr().as_ptr() as *mut i32;- for i in 0..tokens_to_generate {- unsafe {- *ptr.add(i) = (base_position + i) as i32;- }+ let positions = self.positions.as_slice_mut::<i32>();+ for (index, position) in positions.iter_mut().take(tokens_to_generate).enumerate() {+ *position = (base_position + index) as i32;}}/// Prepare seeds buffer with deterministic sequencepub fn prepare_seeds(- &self,+ &mut self,seed: &PRng,prefix_len: usize,tokens_to_generate: usize,) {- let ptr = self.seeds.borrow().cpu_ptr().as_ptr() as *mut u64;- for i in 0..tokens_to_generate {- unsafe {- *ptr.add(i) = seed.derive((prefix_len + i - 1) as u64);- }+ let seeds = self.seeds.as_slice_mut::<u64>();+ for (index, value) in seeds.iter_mut().take(tokens_to_generate).enumerate() {+ *value = seed.derive((prefix_len + index - 1) as u64);}}@@ -114,11 +101,9 @@ pub struct LanguageModelGeneratorContext<B: Backend> {pub context: Rc<B::Context>,pub cache_layers: Rc<RefCell<CacheLayers<B>>>,- pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- pub scratch_buffers: ScratchBuffers<B>,+ pub shared_buffers: Rc<SharedBuffers<B>>,pub model_config: LanguageModelConfig,- pub decoder_config: DecoderConfig,pub model_shape: ModelShape,pub executables: Decoder<B>,pub kv_cache_update: Box<KVCacheUpdate<B>>,@@ -127,7 +112,6 @@ pub struct LanguageModelGeneratorContext<B: Backend> {/// Kernels for copying sampled tokens in async pipelinepub token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,- pub token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,/// Pre-allocated buffers for async generationpub async_buffers: AsyncBuffers<B>,}@@ -159,10 +143,9 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let root_loader_view = loader.tree();- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));- shared_buffers.borrow_mut().update_data(&root_loader_view);-- let scratch_buffers = ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_suffix_length);+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);+ shared_buffers.update_data(&root_loader_view);+ let shared_buffers = Rc::new(shared_buffers);let executables = Decoder::new(context.as_ref(), &decoder_config, &root_loader_view);@@ -180,13 +163,10 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {);let gpu_sampler =- Sampling::<B>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)- .map_err(|e| Error::UnableToCreateContext(e.into()))?;+ Sampling::<B>::new(&context, intermediate_data_type).map_err(|e| Error::UnableToCreateContext(e.into()))?;let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context).map_err(|e| Error::UnableToCreateContext(e.into()))?;- let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)- .map_err(|e| Error::UnableToCreateContext(e.into()))?;let async_batch_size = decoding_config.async_batch_size.resolve::<B>(model_path, context.as_ref());let async_buffers = AsyncBuffers::new(context.as_ref(), max_prefix_length, async_batch_size);@@ -197,16 +177,13 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {context,cache_layers,shared_buffers,- scratch_buffers,model_config: language_model_config.clone(),- decoder_config,model_shape,executables,kv_cache_update,gpu_sampler,seed,token_copy_sampled,- token_copy_results,async_buffers,};diff --git a/crates/backend-uzu/src/lib.rs b/crates/backend-uzu/src/lib.rsindex 8596115c1..bbfa77026 100644--- a/crates/backend-uzu/src/lib.rs+++ b/crates/backend-uzu/src/lib.rs@@ -25,6 +25,7 @@ pub mod session;pub use array::{Array, ArrayContextExt};#[cfg(metal_backend)]pub use audio::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};+pub use backends::common::{AllocationAccessError, allocation_copy_from_slice, allocation_to_vec};pub use config::ConfigDataType;pub use data_type::{ArrayElement, DataType};pub use language_model::gumbel::{gumbel_float, revidx};diff --git a/crates/backend-uzu/src/parameters/loader.rs b/crates/backend-uzu/src/parameters/loader.rsindex 620e19b5a..9bf5c6e86 100644--- a/crates/backend-uzu/src/parameters/loader.rs+++ b/crates/backend-uzu/src/parameters/loader.rs@@ -9,7 +9,7 @@ use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingErroruse crate::{DataType,array::{Array, ArrayContextExt},- backends::common::{Backend, Buffer, Context, DenseBuffer},+ backends::common::{Allocation, AllocationType, AsBufferRangeRef, Backend, Context, DenseBuffer},utils::fs::file_read_exact_at,};@@ -94,27 +94,23 @@ whereself.index.keys()}- pub fn get_leaf<'leaf>(+ fn get_leaf<'leaf>(&'leaf self,key: &str,) -> Result<ParameterLeaf<'file, 'context, 'leaf, C>, ParameterLoaderError<C::Backend>> {Ok(ParameterLeaf {- key: key.to_string(),metadata: self.index.get(key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.to_string()))?,loader: self,})}- pub fn get(+ fn get(&self,key: &str,) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;let (offset, size) = (metadata_entry.offset, metadata_entry.size);- let array_key = key.replace(".", "_");- let array_label = format!("parameter_loader_{array_key}");- let mut array =- self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type, &array_label);+ let mut array = self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type);if array.size() != size {return Err(ParameterLoaderError::SizeMismatch {data_type: metadata_entry.data_type,@@ -151,7 +147,6 @@ where}pub struct ParameterLeaf<'file, 'context, 'leaf, C: Context> {- key: String,metadata: &'leaf ParameterMetadata,loader: &'leaf ParameterLoader<'context, 'file, C>,}@@ -169,16 +164,25 @@ impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, Cself.metadata.size}- pub fn read_buffer(&self) -> Result<<C::Backend as Backend>::DenseBuffer, ParameterLoaderError<C::Backend>> {- let mut buffer =- self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;- buffer.set_label(Some(&format!("parameter_loader_{}", self.key.replace(".", "_"))));+ pub fn read_allocation(&self) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {+ let allocation = self+ .loader+ .context+ .create_allocation(self.metadata.size, AllocationType::Global)+ .map_err(ParameterLoaderError::BackendError)?;+ let buffer_range = allocation.as_buffer_range_ref();+ let range = buffer_range.range();file_read_exact_at(self.loader.file,- unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, self.metadata.size) },+ unsafe {+ std::slice::from_raw_parts_mut(+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),+ range.len(),+ )+ },self.metadata.offset as u64,)?;- Ok(buffer)+ Ok(allocation)}}@@ -188,13 +192,6 @@ pub struct ParameterTree<'loader, C: Context> {}impl<'loader, C: Context> ParameterTree<'loader, C> {- pub fn new(loader: &'loader ParameterLoader<'loader, 'loader, C>) -> Self {- Self {- loader,- prefix: None,- }- }-pub fn path_prefix(&self) -> Option<&str> {self.prefix.as_deref()}@@ -236,6 +233,13 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {self.loader.get_leaf(&self.join_prefix(name))}+ pub fn leaf_allocation(+ &self,+ name: &str,+ ) -> Result<Allocation<C::Backend>, ParameterLoaderError<C::Backend>> {+ self.leaf(name)?.read_allocation()+ }+pub fn read_extract_at(&self,name: &str,@@ -247,14 +251,14 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {}}-pub fn resolve_subtree<'tree, C: Context>(+pub fn try_resolve_subtree<'tree, C: Context>(tree: &'tree ParameterTree<C>,candidates: &[&str],-) -> ParameterTree<'tree, C> {+) -> Result<ParameterTree<'tree, C>, ParameterLoaderError<C::Backend>> {for candidate in candidates {if let Ok(subtree) = tree.subtree(candidate) {- return subtree;+ return Ok(subtree);}}- panic!("Could not find any of {:?} in parameter tree", candidates);+ Err(ParameterLoaderError::SubtreeNotFound(candidates.join(" or ")))}diff --git a/crates/backend-uzu/src/parameters/mod.rs b/crates/backend-uzu/src/parameters/mod.rsindex 67e9f3eaa..a3b5e9087 100644--- a/crates/backend-uzu/src/parameters/mod.rs+++ b/crates/backend-uzu/src/parameters/mod.rs@@ -3,5 +3,5 @@ mod safetensors_metadata;// Re-export the safetensors header reader so other modules (e.g. decoder// runner) can estimate parameter memory before creating a Context.-pub use loader::{ParameterLeaf, ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};+pub use loader::{ParameterLeaf, ParameterLoader, ParameterLoaderError, ParameterTree, try_resolve_subtree};pub use safetensors_metadata::{HeaderLoadingError, read_metadata as read_safetensors_metadata};diff --git a/crates/backend-uzu/src/session/chat_session.rs b/crates/backend-uzu/src/session/chat_session.rsindex b9088bd4e..7915ae225 100644--- a/crates/backend-uzu/src/session/chat_session.rs+++ b/crates/backend-uzu/src/session/chat_session.rs@@ -460,8 +460,9 @@ impl ChatSession {next_to_submit = batch_end;for _ in 0..batch_submitted {- let (_, token, duration_nanos) = receiver.recv().map_err(|_| Error::SamplingFailed)?;+ let (idx, token, duration_nanos) = receiver.recv().map_err(|_| Error::SamplingFailed)?;in_flight -= 1;+ llm.finish_async(idx);let duration = nanos_to_secs(duration_nanos);llm.tokens_push(token);@@ -494,7 +495,8 @@ impl ChatSession {if should_stop {while in_flight > 0 {- let _ = receiver.recv();+ let (idx, _, _) = receiver.recv().map_err(|_| Error::SamplingFailed)?;+ llm.finish_async(idx);in_flight -= 1;}if let Some(slice) = cache_slice {diff --git a/crates/backend-uzu/src/session/classification_session.rs b/crates/backend-uzu/src/session/classification_session.rsindex f863ab7f4..fd243f4dd 100644--- a/crates/backend-uzu/src/session/classification_session.rs+++ b/crates/backend-uzu/src/session/classification_session.rs@@ -13,10 +13,6 @@ use crate::{};pub struct ClassificationSession {- #[allow(dead_code)]- model_path: PathBuf,- #[allow(dead_code)]- model_metadata: ModelMetadata,tokenizer: Tokenizer,classifier: Box<dyn ClassifierTrait>,input_processor: Box<dyn InputProcessor>,@@ -50,8 +46,6 @@ impl ClassificationSession {let classifier = Box::new(Classifier::<B>::new(&model_path, &model_metadata)?);Ok(Self {- model_path,- model_metadata,tokenizer,classifier,input_processor,diff --git a/crates/backend-uzu/src/session/config/decoding_config.rs b/crates/backend-uzu/src/session/config/decoding_config.rsindex d2042126d..27fbd1d57 100644--- a/crates/backend-uzu/src/session/config/decoding_config.rs+++ b/crates/backend-uzu/src/session/config/decoding_config.rs@@ -11,7 +11,6 @@ pub struct DecodingConfig {pub speculator_config: SpeculatorConfig,pub sampling_seed: SamplingSeed,pub async_batch_size: AsyncBatchSize,- pub allow_pre_encode: bool,}impl DecodingConfig {@@ -22,7 +21,6 @@ impl DecodingConfig {speculator_config: SpeculatorConfig,sampling_seed: SamplingSeed,async_batch_size: AsyncBatchSize,- allow_pre_encode: bool,) -> Self {Self {context_mode,@@ -31,7 +29,6 @@ impl DecodingConfig {speculator_config,sampling_seed,async_batch_size,- allow_pre_encode,}}@@ -49,7 +46,6 @@ impl Default for DecodingConfig {speculator_config: SpeculatorConfig::default(),sampling_seed: SamplingSeed::default(),async_batch_size: AsyncBatchSize::default(),- allow_pre_encode: true,}}}@@ -114,14 +110,4 @@ impl DecodingConfig {..self.clone()}}-- pub fn with_allow_pre_encode(- &self,- allow_pre_encode: bool,- ) -> Self {- Self {- allow_pre_encode,- ..self.clone()- }- }}diff --git a/crates/backend-uzu/src/session/tts_session.rs b/crates/backend-uzu/src/session/tts_session.rsindex b1559d1a3..b536d3411 100644--- a/crates/backend-uzu/src/session/tts_session.rs+++ b/crates/backend-uzu/src/session/tts_session.rs@@ -25,7 +25,6 @@ use tokenizers::Tokenizer;use crate::{DataType,- array::ArrayContextExt,audio::{AudioCodecRuntime, AudioGenerationContext, AudioPcmBatch, AudioTokenGrid,nanocodec::{AudioDecodeStepStats, AudioDecodeStreamState},@@ -33,20 +32,14 @@ use crate::{backends::common::{Backend, Context as BackendContext, Encoder, Kernels,kernel::{- EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,- TokenCopyToResultsKernel,+ EmbeddingRowsSumKernel, ManualKernels, TensorAddScaleKernel, TokenCopySampledKernel,kv_cache_update::KVCacheUpdate,matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},},},config::{InnerModelConfig, ModelMetadata, TtsMessageProcessorConfig},encodable_block::{Decoder, EncodingParameters, Sampling as GpuSampling},- forward_pass::{- cache_layers::CacheLayers,- model_shape::ModelShape,- scratch_buffers::ScratchBuffers,- state::{ArrayId, ForwardPassState, SharedBuffers},- },+ forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, state::SharedBuffers},parameters::ParameterLoader,session::{config::{TextDecoderRuntimeConfig, TextSamplingConfig, TtsChunkPolicy, TtsRunConfig, TtsSessionOptions},diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rsindex d4f1bee74..c35872969 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs@@ -1,13 +1,17 @@use super::{decoder_support::*, *};-use crate::{array::Array, backends::common::DenseBuffer, session::types::TtsModelConfigError};+use crate::{+ array::{Array, ArrayContextExt},+ backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},+ encodable_block::{DecoderArguments, DecoderDecodeInput, SamplingInputs},+ forward_pass::token_inputs::TokenInputs,+ session::types::TtsModelConfigError,+};struct TokenDecoderLoadedModel<B: Backend> {- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- scratch_buffers: ScratchBuffers<B>,+ shared_buffers: Rc<SharedBuffers<B>>,executables: Decoder<B>,sampler: GpuSampling<B>,token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,- token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,}impl<B: Backend> TokenDecoderLoadedModel<B> {@@ -19,7 +23,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {transformer_subtree: &str,embedding_subtree: &str,readout_subtree: &str,- max_suffix_length: usize,) -> Result<Self, Error> {let weights_path = model_path.join("model.safetensors");let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;@@ -33,7 +36,6 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {&root_loader_view,transformer_subtree,)?;- let scratch_buffers = ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_suffix_length);let executables = Decoder::new_with_embedding_and_readout_subtrees(context.as_ref(),&decoder_config,@@ -42,22 +44,16 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {embedding_subtree,readout_subtree,);- let logits_data_type = scratch_buffers.logits.data_type();- let sampler =- GpuSampling::new(context.as_ref(), logits_data_type, max_suffix_length, decoder_config.vocab_size)- .map_err(unable_to_create_context)?;+ let logits_data_type = model_shape.activation_data_type();+ let sampler = GpuSampling::new(context.as_ref(), logits_data_type).map_err(unable_to_create_context)?;let token_copy_sampled =<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;- let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(context.as_ref())- .map_err(unable_to_create_context)?;Ok(Self {shared_buffers,- scratch_buffers,executables,sampler,token_copy_sampled,- token_copy_results,})}}@@ -65,8 +61,7 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {struct TokenDecoderContext<B: Backend> {context: Rc<B::Context>,cache_layers: Rc<RefCell<CacheLayers<B>>>,- shared_buffers: Rc<RefCell<SharedBuffers<B>>>,- scratch_buffers: ScratchBuffers<B>,+ shared_buffers: Rc<SharedBuffers<B>>,model_shape: ModelShape,decoder_config: Rc<crate::config::DecoderConfig>,runtime_config: TextDecoderRuntimeConfig,@@ -74,10 +69,9 @@ struct TokenDecoderContext<B: Backend> {sampler: GpuSampling<B>,kv_cache_update: KVCacheUpdate<B>,token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,- token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,- async_chain_positions: Rc<RefCell<B::DenseBuffer>>,- async_chain_seeds: Rc<RefCell<B::DenseBuffer>>,- async_chain_results: Rc<RefCell<B::DenseBuffer>>,+ async_chain_positions: Array<B>,+ async_chain_seeds: Array<B>,+ async_chain_results: Allocation<B>,async_chain_capacity: usize,current_max_prefix_length: usize,}@@ -108,7 +102,6 @@ impl<B: Backend> TokenDecoderContext<B> {transformer_subtree,embedding_subtree,readout_subtree,- max_suffix_length,)?;let async_chain_capacity = max_suffix_length.max(1);let (async_chain_positions, async_chain_seeds, async_chain_results) =@@ -128,7 +121,6 @@ impl<B: Backend> TokenDecoderContext<B> {context,cache_layers,shared_buffers: loaded_model.shared_buffers,- scratch_buffers: loaded_model.scratch_buffers,model_shape,decoder_config,runtime_config: runtime_config.clone(),@@ -136,7 +128,6 @@ impl<B: Backend> TokenDecoderContext<B> {sampler: loaded_model.sampler,kv_cache_update,token_copy_sampled: loaded_model.token_copy_sampled,- token_copy_results: loaded_model.token_copy_results,async_chain_positions,async_chain_seeds,async_chain_results,@@ -153,48 +144,34 @@ impl<B: Backend> TokenDecoderContext<B> {model_shape: &ModelShape,root_loader_view: &crate::parameters::ParameterTree<B::Context>,transformer_subtree: &str,- ) -> Result<Rc<RefCell<SharedBuffers<B>>>, Error> {- let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), decoder_config, model_shape)));+ ) -> Result<Rc<SharedBuffers<B>>, Error> {+ let mut shared_buffers = SharedBuffers::new(context.as_ref(), decoder_config, model_shape);let transformer_tree = root_loader_view.subtree(transformer_subtree).map_err(|_| Error::UnableToLoadWeights)?;- let mut shared_buffers_borrow = shared_buffers.borrow_mut();- if let Some(global_rope) = &mut shared_buffers_borrow.global_rope {+ if let Some(global_rope) = &mut shared_buffers.global_rope {global_rope.update_data(&transformer_tree, "global_rope");}- if let Some(local_rope) = &mut shared_buffers_borrow.local_rope {+ if let Some(local_rope) = &mut shared_buffers.local_rope {local_rope.update_data(&transformer_tree, "local_rope");}- drop(shared_buffers_borrow);- Ok(shared_buffers)+ Ok(Rc::new(shared_buffers))}fn build_async_chain_buffers(context: &Rc<B::Context>,async_chain_capacity: usize,- ) -> Result<(Rc<RefCell<B::DenseBuffer>>, Rc<RefCell<B::DenseBuffer>>, Rc<RefCell<B::DenseBuffer>>), Error> {- let positions = Rc::new(RefCell::new(- context- .create_buffer(async_chain_capacity * std::mem::size_of::<i32>())- .map_err(unable_to_create_context)?,- ));- let seeds = Rc::new(RefCell::new(- context- .create_buffer(async_chain_capacity * std::mem::size_of::<u64>())- .map_err(unable_to_create_context)?,- ));- let results = Rc::new(RefCell::new(- context- .create_buffer(async_chain_capacity * std::mem::size_of::<u32>())- .map_err(unable_to_create_context)?,- ));+ ) -> Result<(Array<B>, Array<B>, Allocation<B>), Error> {+ let positions = context.create_array_uninitialized(&[async_chain_capacity], crate::DataType::I32);+ let seeds = context.create_array_uninitialized(&[async_chain_capacity], crate::DataType::U64);+ let results =+ context.create_array_uninitialized(&[async_chain_capacity], crate::DataType::U32).into_allocation();Ok((positions, seeds, results))}}pub(super) struct TokenDecoderRunner<B: Backend> {ctx: TokenDecoderContext<B>,- pub(super) single_hidden_capture: Array<B>,- pub(super) single_override_embedding: Array<B>,- tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,+ pub(super) single_hidden_capture: Allocation<B>,+ pub(super) single_override_embedding: Allocation<B>,tensor_add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,single_token_vocab_masks: HashMap<usize, Box<[u32]>>,two_token_vocab_masks: HashMap<usize, Box<[u32]>>,@@ -203,6 +180,10 @@ pub(super) struct TokenDecoderRunner<B: Backend> {}impl<B: Backend> TokenDecoderRunner<B> {+ pub(super) fn activation_data_type(&self) -> DataType {+ self.ctx.model_shape.activation_data_type()+ }+pub(super) fn context(&self) -> &Rc<B::Context> {self.ctx.context()}@@ -227,21 +208,17 @@ impl<B: Backend> TokenDecoderRunner<B> {)?;let context: Rc<B::Context> = Rc::clone(ctx.context());let model_dim = ctx.decoder_config.model_dim;- let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(context.as_ref(), activation_data_type)- .map_err(unable_to_create_context)?;let tensor_add_scale =<B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type).map_err(unable_to_create_context)?;- let single_hidden_capture =- context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");+ let single_hidden_capture = context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();let single_override_embedding =- context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_override_embedding");+ context.create_array_zeros(&[1, model_dim], activation_data_type).into_allocation();Ok(Self {ctx,single_hidden_capture,single_override_embedding,- tensor_copy,tensor_add_scale,single_token_vocab_masks: HashMap::new(),two_token_vocab_masks: HashMap::new(),@@ -251,10 +228,110 @@ impl<B: Backend> TokenDecoderRunner<B> {}pub(super) fn reset(&mut self) {- self.ctx.cache_layers.borrow_mut().clear();+ self.ctx.cache_layers.borrow_mut().clear(self.ctx.context.as_ref());self.next_position = 0;}+ fn create_sampling_inputs_from_slices(+ &self,+ token_seeds: &[u64],+ token_bitmask: Option<&[u32]>,+ ) -> SamplingInputs<B> {+ let bitmask_row_len = token_bitmask.map(|_| self.ctx.decoder_config.vocab_size.div_ceil(32));+ SamplingInputs::from_slices(self.ctx.context.as_ref(), token_seeds, token_bitmask, bitmask_row_len)+ }++ fn llm_token_inputs(+ &self,+ token_ids: &[u64],+ token_positions: &[usize],+ token_ids_array: Option<Array<B>>,+ token_positions_array: Option<Array<B>>,+ sampling_start: usize,+ sampling_length: usize,+ ) -> TokenInputs<B> {+ TokenInputs::new_llm(+ self.ctx.context.as_ref(),+ &self.ctx.model_shape,+ token_ids,+ None,+ token_positions,+ token_ids_array,+ token_positions_array,+ sampling_start,+ sampling_length,+ )+ }++ fn decoder_arguments<'a>(+ &'a self,+ token_inputs: &'a TokenInputs<B>,+ batch_dim: usize,+ sampling_start: usize,+ sampling_length: usize,+ cache_layers: &'a mut CacheLayers<B>,+ ) -> DecoderArguments<'a, B> {+ DecoderArguments {+ token_positions: token_inputs.token_positions(),+ token_parents: token_inputs.token_parents(),+ token_subtrie_ranges: token_inputs.token_subtrie_ranges(),+ shared_buffers: self.ctx.shared_buffers.as_ref(),+ cache_layers: Some(cache_layers),+ batch_dim,+ sampling_start,+ sampling_length,+ rope_max_sequence_length: self.ctx.model_shape.context_length(),+ rope_dim: self.ctx.model_shape.rope_dim(),+ #[cfg(feature = "tracing")]+ trace: None,+ }+ }++ fn encode_sampling_on(+ &self,+ encoder: &mut Encoder<B>,+ logits: &mut Allocation<B>,+ output: &mut Allocation<B>,+ sampling_method: SamplingMethod,+ batch_size: usize,+ sampling_inputs: SamplingInputs<B>,+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,+ ) -> Result<(), Error> {+ pending_sampling_inputs.push(sampling_inputs);+ let sampling_inputs = pending_sampling_inputs.last().expect("sampling inputs must be pending");+ self.ctx+ .sampler+ .encode(+ crate::encodable_block::SamplingArguments {+ logits,+ seeds: sampling_inputs.seeds.allocation(),+ bitmask: sampling_inputs.bitmask.as_ref().map(Array::allocation),+ output,+ sampling_method,+ batch_size,+ vocab_size: self.ctx.decoder_config.vocab_size,+ },+ encoder,+ )+ .map_err(|err| Error::EncodeFailed(Box::new(err)))+ }++ fn copy_sampling_output_to_async_result_on(+ &mut self,+ encoder: &mut Encoder<B>,+ sampling_output: &Allocation<B>,+ result_slot: usize,+ ) {+ let copy_size = std::mem::size_of::<u32>();+ let result_offset = result_slot * copy_size;+ encoder.encode_copy(+ sampling_output,+ 0..copy_size,+ &mut self.ctx.async_chain_results,+ result_offset..result_offset + copy_size,+ );+ }+pub(super) fn prepare_for_generation(&mut self,max_prefix_length: usize,@@ -286,44 +363,24 @@ impl<B: Backend> TokenDecoderRunner<B> {let token_count = token_ids.len();let positions: Vec<usize> = (self.next_position..self.next_position + token_count).collect();- let token_seeds = vec![0_u64; token_count];-- let mut state = ForwardPassState::new_llm(- self.ctx.context.clone(),- &self.ctx.decoder_config,- &self.ctx.model_shape,- &self.ctx.scratch_buffers,- self.ctx.cache_layers.clone(),- self.ctx.shared_buffers.clone(),- token_ids,- None,- &positions,- None, // no bitmask- &token_seeds,- token_count,- 0, // sampling_start: irrelevant- 0, // sampling_length: no sampling- false,- false,- None,- None,- );+ let token_inputs = self.llm_token_inputs(token_ids, &positions, None, None, 0, 0);let encoding_parameters = EncodingParameters::new();let context = Rc::clone(&self.ctx.context);let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;+ let mut cache_layers = self.ctx.cache_layers.borrow_mut();self.ctx.executables- .embed- .encode_lookup(&mut state, &mut encoder)+ .encode_prefill(+ self.decoder_arguments(&token_inputs, token_count, 0, 0, &mut *cache_layers),+ token_inputs.token_ids(),+ &encoding_parameters,+ &mut encoder,+ ).map_err(|err| Error::EncodeFailed(Box::new(err)))?;- for layer in self.ctx.executables.layers.iter() {- layer- .encode(&mut state, &encoding_parameters, &mut encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;- }+ drop(cache_layers);self.encode_cache_acceptance_update_on(&mut encoder, token_count);self.submit_and_wait_command_buffer(encoder)?;@@ -354,10 +411,13 @@ impl<B: Backend> TokenDecoderRunner<B> {encoder: &mut Encoder<B>,initial_token_ids: &[u64],initial_embedding_injection: EmbeddingInjection,- pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,followup_count: usize,vocab_limit: Option<usize>,sampling: &mut TextSamplingState,+ pending_token_inputs: &mut Vec<TokenInputs<B>>,+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,+ pending_sampling_outputs: &mut Vec<Allocation<B>>,) -> Result<usize, Error> {let total_count =1_usize.checked_add(followup_count).ok_or(TtsModelConfigError::AsyncChainFollowupCountOverflow)?;@@ -416,34 +476,24 @@ impl<B: Backend> TokenDecoderRunner<B> {// Pre-fill followup positions and seeds into the async chain buffers.let followup_base_position = self.next_position + initial_token_ids.len();{- let positions_borrow = self.ctx.async_chain_positions.borrow();- let positions_ptr = positions_borrow.cpu_ptr().as_ptr() as *mut i32;+ let positions = self.ctx.async_chain_positions.as_slice_mut::<i32>();for pass in 0..followup_count {- unsafe {- *positions_ptr.add(pass) = (followup_base_position + pass) as i32;- }+ positions[pass] = (followup_base_position + pass) as i32;}}{- let seeds_borrow = self.ctx.async_chain_seeds.borrow();- let seeds_ptr = seeds_borrow.cpu_ptr().as_ptr() as *mut u64;+ let seeds = self.ctx.async_chain_seeds.as_slice_mut::<u64>();if !matches!(sampling.method(), SamplingMethod::Greedy) {- for pass in 0..followup_count {- unsafe {- *seeds_ptr.add(pass) = sampling.next_seed();- }+ for seed in seeds.iter_mut().take(followup_count) {+ *seed = sampling.next_seed();}} else {- for pass in 0..followup_count {- unsafe {- *seeds_ptr.add(pass) = 0;- }- }+ seeds[..followup_count].fill(0);}}// Encode the initial forward pass (no new CB, no submit).- let state = self.encode_single_forward_pass_on(+ let sampling_output = self.encode_single_forward_pass_on(encoder,initial_token_ids,initial_embedding_injection,@@ -452,30 +502,41 @@ impl<B: Backend> TokenDecoderRunner<B> {false, // capture_hiddenpre_injection_encode,initial_seed,+ pending_token_inputs,+ pending_sampling_inputs,)?;- // Copy the sampled token for chaining and into results[0].- {- let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;- let sampling_output_buffer = sampling_output.buffer();- let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();-- let sampling_output_buffer = sampling_output_buffer.borrow();- let mut token_ids_buffer = token_ids_buffer.borrow_mut();- if followup_count > 0 {- self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);- }- let mut results_buffer = self.ctx.async_chain_results.borrow_mut();- self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);- }+ let first_followup_token_ids = if followup_count > 0 {+ let token_ids_shape = [1];+ let token_ids_data_type = crate::DataType::U64;+ let mut next_token_ids =+ self.ctx.context.create_array_uninitialized(&token_ids_shape, token_ids_data_type).into_allocation();+ self.ctx.token_copy_sampled.encode(&sampling_output, &mut next_token_ids, encoder);+ Some(unsafe { Array::from_allocation(next_token_ids, 0, &token_ids_shape, token_ids_data_type) })+ } else {+ None+ };+ self.copy_sampling_output_to_async_result_on(encoder, &sampling_output, 0);self.encode_cache_acceptance_update_on(encoder, initial_token_count);self.register_positions_and_advance(initial_token_count);if followup_count > 0 {- self.encode_followup_passes_on(encoder, followup_count, 0, vocab_mask_limit, sampling.method(), 1)?;+ self.encode_followup_passes_on(+ encoder,+ followup_count,+ first_followup_token_ids,+ vocab_mask_limit,+ sampling.method(),+ 1,+ pending_token_inputs,+ pending_sampling_inputs,+ pending_sampling_outputs,+ )?;}+ pending_sampling_outputs.push(sampling_output);+self.next_position = followup_base_position.saturating_add(followup_count);// Do NOT submit -- the caller will submit after encoding more work.Ok(total_count)@@ -493,91 +554,69 @@ impl<B: Backend> TokenDecoderRunner<B> {&mut self,encoder: &mut Encoder<B>,followup_count: usize,- first_token: u64,+ first_token_ids: Option<Array<B>>,vocab_mask_limit: Option<usize>,sampling_method: SamplingMethod,results_offset_slots: usize,+ pending_token_inputs: &mut Vec<TokenInputs<B>>,+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,+ pending_sampling_outputs: &mut Vec<Allocation<B>>,) -> Result<(), Error> {let encoding_parameters = EncodingParameters::new();- let context_rc = self.ctx.context.clone();let cache_layers_rc = self.ctx.cache_layers.clone();- let shared_buffers_rc = self.ctx.shared_buffers.clone();- let positions_rc = self.ctx.async_chain_positions.clone();- let seeds_rc = self.ctx.async_chain_seeds.clone();- let token_bitmask = vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit));+ let token_bitmask =+ vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit)).map(Box::<[u32]>::from);+ let mut next_token_ids = first_token_ids;for pass in 0..followup_count {let results_slot = results_offset_slots + pass;- let token_ids = [if pass == 0 {- first_token- } else {- 0- }];- let mut state = ForwardPassState::new_llm(- context_rc.clone(),- &self.ctx.decoder_config,- &self.ctx.model_shape,- &self.ctx.scratch_buffers,- cache_layers_rc.clone(),- shared_buffers_rc.clone(),- &token_ids,- None,- &[self.next_position + pass],- token_bitmask,- &[0],- 1,- 0,- 1,- false,- pass > 0 || results_offset_slots > 0,- Some((positions_rc.clone(), pass)),- Some((seeds_rc.clone(), pass)),- );- if let Some(method) = state.sampling_method_mut() {- *method = Some(sampling_method);- }-- {- self.ctx- .executables- .embed- .encode_lookup(&mut state, encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;- for layer in self.ctx.executables.layers.iter() {- layer- .encode(&mut state, &encoding_parameters, encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;- }- self.ctx- .executables- .norm- .encode(&mut state, encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;- self.ctx- .executables- .embed- .encode_readout(&mut state, encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;- self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;- }-- let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;- let sampling_output_buffer = sampling_output.buffer();- let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();+ let token_ids = [0];+ let seed = self.ctx.async_chain_seeds.as_slice::<u64>()[pass];+ let sampling_inputs = self.create_sampling_inputs_from_slices(&[seed], token_bitmask.as_deref());+ let token_inputs =+ self.llm_token_inputs(&token_ids, &[self.next_position + pass], next_token_ids.take(), None, 0, 1);+ pending_token_inputs.push(token_inputs);+ let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");{- let sampling_output_buffer = sampling_output_buffer.borrow();- let mut token_ids_buffer = token_ids_buffer.borrow_mut();+ let mut logits = {+ let mut cache_layers = cache_layers_rc.borrow_mut();+ self.ctx+ .executables+ .encode_decode(+ self.decoder_arguments(&token_inputs, 1, 0, 1, &mut *cache_layers),+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),+ None,+ &encoding_parameters,+ encoder,+ )+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?+ };+ let mut sampling_output = self.create_sampling_output(1);+ self.encode_sampling_on(+ encoder,+ &mut logits,+ &mut sampling_output,+ sampling_method,+ 1,+ sampling_inputs,+ pending_sampling_inputs,+ )?;if pass + 1 < followup_count {- self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);+ let token_ids_shape = [1];+ let token_ids_data_type = crate::DataType::U64;+ let mut updated_token_ids = self+ .ctx+ .context+ .create_array_uninitialized(&token_ids_shape, token_ids_data_type)+ .into_allocation();+ self.ctx.token_copy_sampled.encode(&sampling_output, &mut updated_token_ids, encoder);+ next_token_ids = Some(unsafe {+ Array::from_allocation(updated_token_ids, 0, &token_ids_shape, token_ids_data_type)+ });}- let results_offset = results_slot * std::mem::size_of::<u32>();- let mut results_buffer = self.ctx.async_chain_results.borrow_mut();- self.ctx.token_copy_results.encode(- &*sampling_output_buffer,- (&mut *results_buffer, results_offset),- encoder,- );+ self.copy_sampling_output_to_async_result_on(encoder, &sampling_output, results_slot);+ pending_sampling_outputs.push(sampling_output);}self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);@@ -647,6 +686,13 @@ impl<B: Backend> TokenDecoderRunner<B> {self.two_token_vocab_masks.get(&vocab_limit).map(|mask| mask.as_ref())}+ fn create_sampling_output(+ &self,+ sampling_length: usize,+ ) -> Allocation<B> {+ self.ctx.context.create_array_uninitialized(&[sampling_length], crate::DataType::U32).into_allocation()+ }+fn encode_single_forward_pass_on(&mut self,encoder: &mut Encoder<B>,@@ -655,9 +701,11 @@ impl<B: Backend> TokenDecoderRunner<B> {sampling: &mut TextSamplingState,token_bitmask: Option<&[u32]>,capture_hidden: bool,- mut pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ mut pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,preconsumed_seed: Option<u64>,- ) -> Result<ForwardPassState<B>, Error> {+ pending_token_inputs: &mut Vec<TokenInputs<B>>,+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,+ ) -> Result<Allocation<B>, Error> {if token_ids.is_empty() {return Err(Error::GenerateFailed);}@@ -715,68 +763,80 @@ impl<B: Backend> TokenDecoderRunner<B> {return Err(Error::GenerateFailed);}- let mut state = ForwardPassState::new_llm(- self.ctx.context.clone(),- &self.ctx.decoder_config,- &self.ctx.model_shape,- &self.ctx.scratch_buffers,- self.ctx.cache_layers.clone(),- self.ctx.shared_buffers.clone(),- token_ids,- None,- positions,- token_bitmask,- token_seeds,- token_count,- sampling_start,- sampling_length,- false,- false,- None,- None,- );- if let Some(method) = state.sampling_method_mut() {- *method = Some(sampling.method());- }+ let sampling_seed_end = sampling_start + sampling_length;+ let sampling_token_seeds = &token_seeds[sampling_start..sampling_seed_end];+ let sampling_token_bitmask = token_bitmask.map(|mask| {+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);+ let row_start = sampling_start * row_words;+ let row_end = row_start + sampling_length * row_words;+ &mask[row_start..row_end]+ });+ let sampling_inputs = self.create_sampling_inputs_from_slices(sampling_token_seeds, sampling_token_bitmask);+ let token_inputs = self.llm_token_inputs(token_ids, positions, None, None, sampling_start, sampling_length);+ pending_token_inputs.push(token_inputs);+ let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");let encoding_parameters = EncodingParameters::new();- self.ctx+ let mut main = self+ .ctx.executables.embed- .encode_lookup(&mut state, encoder)+ .encode_lookup(token_inputs.token_ids(), token_count, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;if let Some(pre_encode) = pre_injection_encode.as_mut() {- pre_encode(self, &state, encoder)?;+ pre_encode(self, encoder)?;}match embedding_injection {EmbeddingInjection::None => {},EmbeddingInjection::AddPreloaded {post_scale,} => {- self.encode_add_scale_from_single_bias_on(encoder, &state, token_count, post_scale.unwrap_or(1.0))?;+ self.encode_add_scale_from_single_bias_on(encoder, &mut main, token_count, post_scale.unwrap_or(1.0))?;},EmbeddingInjection::OverrideFirstRowInternal => {- self.encode_override_first_row_from_device_on(encoder, &state, &self.single_override_embedding)?;+ self.encode_override_first_row_from_device_on(encoder, &mut main, &self.single_override_embedding)?;},}- for layer in self.ctx.executables.layers.iter() {- layer- .encode(&mut state, &encoding_parameters, encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;- }- self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;- if capture_hidden {- // After output norm with residual_add, Shortcut holds the un-normalized full residual- self.encode_capture_last_hidden_into_single_buffer_on(encoder, &state, token_count)?;- }- self.ctx- .executables- .embed- .encode_readout(&mut state, encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?;- self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ let mut logits = {+ let mut cache_layers = self.ctx.cache_layers.borrow_mut();+ let decoder_arguments = DecoderArguments {+ token_positions: token_inputs.token_positions(),+ token_parents: token_inputs.token_parents(),+ token_subtrie_ranges: token_inputs.token_subtrie_ranges(),+ shared_buffers: self.ctx.shared_buffers.as_ref(),+ cache_layers: Some(&mut *cache_layers),+ batch_dim: token_count,+ sampling_start,+ sampling_length,+ rope_max_sequence_length: self.ctx.model_shape.context_length(),+ rope_dim: self.ctx.model_shape.rope_dim(),+ #[cfg(feature = "tracing")]+ trace: None,+ };+ let hidden_capture = capture_hidden.then_some(&mut self.single_hidden_capture);+ self.ctx+ .executables+ .encode_decode(+ decoder_arguments,+ DecoderDecodeInput::Embeddings(main),+ hidden_capture,+ &encoding_parameters,+ encoder,+ )+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?+ };+ let mut sampling_output = self.create_sampling_output(sampling_length);+ self.encode_sampling_on(+ encoder,+ &mut logits,+ &mut sampling_output,+ sampling.method(),+ sampling_length,+ sampling_inputs,+ pending_sampling_inputs,+ )?;- Ok(state)+ Ok(sampling_output)}/// Encode the KV-cache acceptance update for `token_count` tokens onto@@ -835,7 +895,7 @@ impl<B: Backend> TokenDecoderRunner<B> {sampling: &mut TextSamplingState,precomputed_token_bitmask: Option<&[u32]>,capture_hidden: bool,- pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,) -> Result<u64, Error> {objc2::rc::autoreleasepool(|_| {if token_ids.is_empty() {@@ -910,8 +970,10 @@ impl<B: Backend> TokenDecoderRunner<B> {let context = Rc::clone(&self.ctx.context);let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;+ let mut pending_token_inputs = Vec::new();+ let mut pending_sampling_inputs = Vec::new();- let state = self.encode_single_forward_pass_on(+ let sampling_output = self.encode_single_forward_pass_on(&mut encoder,token_ids,embedding_injection,@@ -920,97 +982,38 @@ impl<B: Backend> TokenDecoderRunner<B> {capture_hidden,pre_injection_encode,None,+ &mut pending_token_inputs,+ &mut pending_sampling_inputs,)?;self.encode_cache_acceptance_update_on(&mut encoder, token_count);self.submit_and_wait_command_buffer(encoder)?;- let token = read_sampled_token_from_sampling_output(&state)?;+ let token = read_sampled_token_from_sampling_output(&sampling_output)?;self.register_positions_and_advance(token_count);Ok(token)})}- fn encode_capture_last_hidden_into_single_buffer_on(- &self,- encoder: &mut Encoder<B>,- state: &ForwardPassState<B>,- token_count: usize,- ) -> Result<(), Error> {- if token_count == 0 {- return Err(Error::GenerateFailed);- }- let model_dim = self.ctx.decoder_config.model_dim;- let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {- model_dim,- })?;- let shortcut = state.array(ArrayId::Shortcut);- let bytes_per_element = shortcut.data_type().size_in_bytes();- let row_offset = (token_count - 1)- .checked_mul(model_dim)- .and_then(|value| value.checked_mul(bytes_per_element))- .ok_or(TtsModelConfigError::HiddenCaptureRowOffsetOverflow {- token_count,- model_dim,- })?;- let src_offset =- shortcut.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;- let capture = &self.single_hidden_capture;- if capture.shape() != [1, model_dim] || capture.data_type() != shortcut.data_type() {- return Err(TtsModelConfigError::HiddenCaptureTensorMismatch {- expected_shape: [1, model_dim].into(),- expected_data_type: shortcut.data_type(),- actual_shape: capture.shape().into(),- actual_data_type: capture.data_type(),- }- .into());- }-- let shortcut_buffer = shortcut.buffer();- let shortcut_buffer = shortcut_buffer.borrow();- let capture_buffer = capture.buffer();- let mut capture_buffer = capture_buffer.borrow_mut();- self.tensor_copy.encode((&*shortcut_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);- Ok(())- }-fn encode_override_first_row_from_device_on(&self,encoder: &mut Encoder<B>,- state: &ForwardPassState<B>,- override_embedding: &Array<B>,+ main: &mut Allocation<B>,+ override_embedding: &Allocation<B>,) -> Result<(), Error> {let model_dim = self.ctx.decoder_config.model_dim;- let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {- model_dim,- })?;- let main = state.array(ArrayId::Main);- if override_embedding.shape() != [1, model_dim] || override_embedding.data_type() != main.data_type() {- return Err(TtsModelConfigError::OverrideEmbeddingTensorMismatch {- expected_shape: [1, model_dim].into(),- expected_data_type: main.data_type(),- actual_shape: override_embedding.shape().into(),- actual_data_type: override_embedding.data_type(),- }- .into());- }-- let override_buffer = override_embedding.buffer();- let override_buffer = override_buffer.borrow();- let main_buffer = main.buffer();- let mut main_buffer = main_buffer.borrow_mut();- self.tensor_copy.encode(- (&*override_buffer, override_embedding.offset()),- (&mut *main_buffer, main.offset()),- model_dim_u32,- encoder,- );+ let model_dim_bytes = model_dim.checked_mul(self.activation_data_type().size_in_bytes()).ok_or(+ TtsModelConfigError::ModelDimExceedsU32 {+ model_dim,+ },+ )?;+ encoder.encode_copy(override_embedding, 0..model_dim_bytes, main, 0..model_dim_bytes);Ok(())}fn encode_add_scale_from_single_bias_on(&self,encoder: &mut Encoder<B>,- state: &ForwardPassState<B>,+ main: &mut Allocation<B>,token_count: usize,scale: f32,) -> Result<(), Error> {@@ -1029,34 +1032,18 @@ impl<B: Backend> TokenDecoderRunner<B> {u32::try_from(total_len).map_err(|_| TtsModelConfigError::AddScaleTotalLengthExceedsU32 {total_len,})?;-- let main = state.array(ArrayId::Main);- let bias = &self.single_override_embedding;- if bias.shape() != [1, model_dim] || bias.data_type() != main.data_type() {- return Err(TtsModelConfigError::AddScaleBiasTensorMismatch {- expected_shape: [1, model_dim].into(),- expected_data_type: main.data_type(),- actual_shape: bias.shape().into(),- actual_data_type: bias.data_type(),- }- .into());- }-- let bias_buffer = bias.buffer();- let bias_buffer = bias_buffer.borrow();- let main_output_buffer = main.buffer();- let mut main_output_buffer = main_output_buffer.borrow_mut();- // TensorAddScale is elementwise, so in-place read/write aliasing is valid here.- let main_input_buffer: &B::DenseBuffer = unsafe { &*(&*main_output_buffer as *const B::DenseBuffer) };+ let mut output =+ encoder.allocate_scratch(main.as_buffer_range_ref().range().len()).map_err(unable_to_create_context)?;self.tensor_add_scale.encode(- (main_input_buffer, main.offset()),- &*bias_buffer,- (&mut *main_output_buffer, main.offset()),+ &*main,+ &self.single_override_embedding,+ &mut output,model_dim_u32,total_len_u32,scale,encoder,);+ *main = output;Ok(())}@@ -1075,11 +1062,14 @@ impl<B: Backend> TokenDecoderRunner<B> {sampling: &mut TextSamplingState,precomputed_token_bitmask: Option<&[u32]>,capture_hidden: bool,- pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<B>>,+ pending_token_inputs: &mut Vec<TokenInputs<B>>,+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,+ pending_sampling_outputs: &mut Vec<Allocation<B>>,) -> Result<(), Error> {let token_count = token_ids.len();- let state = self.encode_single_forward_pass_on(+ let sampling_output = self.encode_single_forward_pass_on(encoder,token_ids,embedding_injection,@@ -1088,18 +1078,14 @@ impl<B: Backend> TokenDecoderRunner<B> {capture_hidden,pre_injection_encode,None,+ pending_token_inputs,+ pending_sampling_inputs,)?;// Copy sampled token to async_chain_results[0] so the caller can// read it after the eventual submit.- {- let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;- let sampling_output_buffer = sampling_output.buffer();-- let sampling_output_buffer = sampling_output_buffer.borrow();- let mut results_buffer = self.ctx.async_chain_results.borrow_mut();- self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);- }+ self.copy_sampling_output_to_async_result_on(encoder, &sampling_output, 0);+ pending_sampling_outputs.push(sampling_output);self.encode_cache_acceptance_update_on(encoder, token_count);// Do NOT submit -- the caller will submit.@@ -1112,20 +1098,22 @@ impl<B: Backend> TokenDecoderRunner<B> {encoder: Encoder<B>,) -> Result<(), Error> {self.instrumentation.command_buffers_submitted += 1;- encoder+ let result = encoder.end_encoding().submit().wait_until_completed()- .map_err(|err| Error::CommandBufferFailed(Box::new(err)))?;+ .map_err(|err| Error::CommandBufferFailed(Box::new(err)));+ result?;self.instrumentation.host_waits += 1;Ok(())}pub(super) fn copy_async_chain_results_to_on(- &self,+ &mut self,encoder: &mut Encoder<B>,src_slot: usize,- dst: (&mut B::DenseBuffer, usize),+ dst: &mut Allocation<B>,+ dst_offset: usize,count: usize,) -> Result<(), Error> {let src_offset = src_slot.checked_mul(std::mem::size_of::<u32>()).ok_or(@@ -1133,12 +1121,16 @@ impl<B: Backend> TokenDecoderRunner<B> {src_slot,},)?;- let copy_size =+ let byte_len =count.checked_mul(std::mem::size_of::<u32>()).ok_or(TtsModelConfigError::AsyncChainCopySizeOverflow {count,})?;- let results_buffer = self.ctx.async_chain_results.borrow();- encoder.encode_copy(&results_buffer, src_offset..src_offset + copy_size, dst.0, dst.1..dst.1 + copy_size);+ encoder.encode_copy(+ &self.ctx.async_chain_results,+ src_offset..src_offset + byte_len,+ dst,+ dst_offset..dst_offset + byte_len,+ );Ok(())}@@ -1146,8 +1138,6 @@ impl<B: Backend> TokenDecoderRunner<B> {&self,slot: usize,) -> Result<u32, Error> {- let results_buffer = self.ctx.async_chain_results.borrow();- let ptr = results_buffer.cpu_ptr().as_ptr() as *const u32;let capacity = self.ctx.async_chain_capacity;if slot >= capacity {return Err(TtsModelConfigError::AsyncChainResultSlotOutOfBounds {@@ -1156,7 +1146,12 @@ impl<B: Backend> TokenDecoderRunner<B> {}.into());}- Ok(unsafe { *ptr.add(slot) })+ let buffer_range = self.ctx.async_chain_results.as_buffer_range_ref();+ let range = buffer_range.range();+ Ok(unsafe {+ *((buffer_range.buffer().cpu_ptr().as_ptr() as *const u8)+ .add(range.start + slot * std::mem::size_of::<u32>()) as *const u32)+ })}pub(super) fn take_instrumentation(&mut self) -> RunnerInstrumentation {@@ -1168,9 +1163,9 @@ impl<B: Backend> TokenDecoderRunner<B> {}}-fn read_sampled_token_from_sampling_output<B: Backend>(state: &ForwardPassState<B>) -> Result<u64, Error> {- let output = state.sampling_output().ok_or(Error::SamplingFailed)?;- let tokens = output.as_slice::<u32>();- let token = tokens.first().copied().ok_or(Error::SamplingFailed)?;+fn read_sampled_token_from_sampling_output<B: Backend>(sampling_output: &Allocation<B>) -> Result<u64, Error> {+ let buffer_range = sampling_output.as_buffer_range_ref();+ let range = buffer_range.range();+ let token = unsafe { *((buffer_range.buffer().cpu_ptr().as_ptr() as *const u8).add(range.start) as *const u32) };Ok(u64::from(token))}diff --git a/crates/backend-uzu/src/session/tts_session/decoder_support.rs b/crates/backend-uzu/src/session/tts_session/decoder_support.rsindex 8b708f6da..68cdb2653 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_support.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_support.rs@@ -1,7 +1,7 @@use super::*;pub(super) type PreInjectionEncodeCallback<'a, B> =- dyn FnMut(&TokenDecoderRunner<B>, &ForwardPassState<B>, &mut Encoder<B>) -> Result<(), Error> + 'a;+ dyn FnMut(&mut TokenDecoderRunner<B>, &mut Encoder<B>) -> Result<(), Error> + 'a;pub(super) enum EmbeddingInjection {None,@@ -362,7 +362,7 @@ impl<'a, F: FnMut(&AudioPcmBatch)> StreamingSynthesisState<'a, F> {let ready_frames = pending.ready_frames;let next_chunk_frames = pending.next_chunk_frames;let submission_decode_duration = pending.submission_decode_duration;- let _ = self.pending_chunk.take().ok_or(Error::GenerateFailed)?;+ self.pending_chunk.take().ok_or(Error::GenerateFailed)?;(step_stats,ready_frames,diff --git a/crates/backend-uzu/src/session/tts_session/fishaudio.rs b/crates/backend-uzu/src/session/tts_session/fishaudio.rsindex 0e4cbd902..ef1b5dc55 100644--- a/crates/backend-uzu/src/session/tts_session/fishaudio.rs+++ b/crates/backend-uzu/src/session/tts_session/fishaudio.rs@@ -4,8 +4,11 @@ use regex::Regex;use super::*;use crate::{- array::Array,+ array::ArrayContextExt,+ backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},config::TtsMessageProcessorConfig,+ encodable_block::SamplingInputs,+ forward_pass::token_inputs::TokenInputs,session::types::{TtsModelConfigError, TtsPromptConfigError},};@@ -35,10 +38,13 @@ pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {struct FishAudioSemanticBridge<B: Backend> {embedding_rows_sum: <B::Kernels as Kernels>::EmbeddingRowsSumKernel,projection: <B::Kernels as ManualKernels>::MatmulKernel,- tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,- codebook_embeddings: Array<B>,- codebook_token_indices: Array<B>,- projection_weights: Option<Array<B>>,+ codebook_embeddings: Allocation<B>,+ codebook_token_indices: Allocation<B>,+ projection_weights: Option<Allocation<B>>,+ num_codebooks: usize,+ codebook_size: usize,+ slow_model_dim: usize,+ fast_model_dim: usize,}impl<B: Backend> FishAudioSemanticBridge<B> {@@ -48,7 +54,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {config: &crate::config::FishAudioTextDecoderConfig,data_type: DataType,) -> Result<Self, Error> {- let codebook_embeddings = load_float_tensor_array(+ let codebook_embeddings = load_float_tensor_allocation(parameter_tree,"text_decoder.codebook_embeddings.weights",[@@ -58,7 +64,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {data_type,)?;let fast_model_projection = if config.fast_model_projection_config.is_some() {- Some(load_float_tensor_array(+ Some(load_float_tensor_allocation(parameter_tree,"text_decoder.fast_model_projection.weights",[config.fast_model_dim, config.slow_model_dim],@@ -83,64 +89,52 @@ impl<B: Backend> FishAudioSemanticBridge<B> {fn new(context: &B::Context,data_type: DataType,- codebook_embeddings: Array<B>,- fast_model_projection: Option<Array<B>>,+ codebook_embeddings: Allocation<B>,+ fast_model_projection: Option<Allocation<B>>,num_codebooks: usize,codebook_size: usize,slow_model_dim: usize,fast_model_dim: usize,) -> Result<Self, Error> {- if codebook_embeddings.shape()- != [num_codebooks.checked_mul(codebook_size).ok_or(Error::UnableToLoadConfig)?, slow_model_dim]- || codebook_embeddings.data_type() != data_type- {- return Err(Error::UnableToLoadConfig);- }- if let Some(ref projection) = fast_model_projection- && (projection.shape() != [fast_model_dim, slow_model_dim] || projection.data_type() != data_type)- {- return Err(Error::UnableToLoadConfig);- }-let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type).map_err(unable_to_create_context)?;let projection = <<B::Kernels as ManualKernels>::MatmulKernel as MatmulKernel>::new(context, data_type).map_err(unable_to_create_context)?;- let tensor_copy =- <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;- let codebook_token_indices =- context.create_array_zeros(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");+ let codebook_token_indices = context.create_array_zeros(&[num_codebooks], DataType::U32).into_allocation();Ok(Self {embedding_rows_sum,projection,- tensor_copy,codebook_embeddings,codebook_token_indices,projection_weights: fast_model_projection,+ num_codebooks,+ codebook_size,+ slow_model_dim,+ fast_model_dim,})}}-fn load_float_tensor_array<B: Backend>(+fn load_float_tensor_allocation<B: Backend>(parameter_tree: &crate::parameters::ParameterTree<B::Context>,key: &str,expected_shape: [usize; 2],target_data_type: DataType,-) -> Result<Array<B>, Error> {- let array = parameter_tree.leaf_array(key).map_err(|_| Error::UnableToLoadWeights)?;- if array.shape() != expected_shape {+) -> Result<Allocation<B>, Error> {+ let leaf = parameter_tree.leaf(key).map_err(|_| Error::UnableToLoadWeights)?;+ if leaf.shape() != expected_shape {return Err(Error::UnableToLoadConfig);}- if array.data_type() != target_data_type {+ if leaf.data_type() != target_data_type {return Err(TtsModelConfigError::FishAudioTensorDataTypeMismatch {key: key.into(),expected: target_data_type,- actual: array.data_type(),+ actual: leaf.data_type(),}.into());}- Ok(array)+ leaf.read_allocation().map_err(|_| Error::UnableToLoadWeights)}fn validate_fishaudio_decoder_contract(@@ -303,8 +297,8 @@ pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(runtime_config,)?;- let activation_data_type = slow_runner.single_hidden_capture.data_type();- let fast_data_type = fast_runner.single_override_embedding.data_type();+ let activation_data_type = slow_runner.activation_data_type();+ let fast_data_type = fast_runner.activation_data_type();if fast_data_type != activation_data_type {return Err(Error::UnableToLoadConfig);}@@ -510,42 +504,25 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {}fn encode_project_slow_hidden_to_fast_on(- context: &B::Context,semantic_bridge: &mut FishAudioSemanticBridge<B>,- slow_hidden_capture: &Array<B>,- output_embedding: &Array<B>,+ slow_hidden_capture: &Allocation<B>,+ output_embedding: &mut Allocation<B>,slow_model_dim: usize,fast_model_dim: usize,encoder: &mut Encoder<B>,) -> Result<(), Error> {- let model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;-if let Some(weights) = semantic_bridge.projection_weights.as_ref() {- let hidden = slow_hidden_capture;- let weights = weights;- let output = output_embedding;- if hidden.shape() != [1, slow_model_dim]- || output.shape() != [1, fast_model_dim]- || weights.shape() != [fast_model_dim, slow_model_dim]- {+ if semantic_bridge.fast_model_dim != fast_model_dim || semantic_bridge.slow_model_dim != slow_model_dim {return Err(Error::GenerateFailed);}-- let hidden_buffer = hidden.buffer();- let weights_buffer = weights.buffer();- let output_buffer = output.buffer();- let hidden_buffer = hidden_buffer.borrow();- let weights_buffer = weights_buffer.borrow();- let mut output_buffer = output_buffer.borrow_mut();semantic_bridge.projection.encode(- context,MatmulArguments {- a: &hidden_buffer,- a_offset: hidden.offset() as u64,- b: &weights_buffer,+ a: slow_hidden_capture,+ a_offset: 0,+ b: weights,ab_scale: 1.0,c: MatmulArgumentC::None,- d: &mut output_buffer,+ d: output_embedding,batch_dim: 1,input_dim: slow_model_dim as u32,output_dim: fast_model_dim as u32,@@ -559,21 +536,11 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {return Err(Error::UnableToLoadConfig);}- let hidden = slow_hidden_capture;- let output = output_embedding;- if hidden.shape() != [1, slow_model_dim] || output.shape() != [1, fast_model_dim] {- return Err(Error::GenerateFailed);- }- let hidden_buffer = hidden.buffer();- let hidden_buffer = hidden_buffer.borrow();- let output_buffer = output.buffer();- let mut output_buffer = output_buffer.borrow_mut();- semantic_bridge.tensor_copy.encode(- (&*hidden_buffer, hidden.offset()),- (&mut *output_buffer, output.offset()),- model_dim_u32,- encoder,+ debug_assert_eq!(+ slow_hidden_capture.as_buffer_range_ref().range().len(),+ output_embedding.as_buffer_range_ref().range().len());+ encoder.encode_copy(slow_hidden_capture, .., output_embedding, ..);Ok(())}@@ -585,9 +552,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {/// `gpu_tokens_buffer` at the given byte offset.fn encode_slow_codebook_sum_from_gpu_tokens_on(semantic_bridge: &mut FishAudioSemanticBridge<B>,- slow_sum_embedding: &Array<B>,+ slow_sum_embedding: &mut Allocation<B>,first_code: u32,- source_runner: &TokenDecoderRunner<B>,+ source_runner: &mut TokenDecoderRunner<B>,source_slot: usize,num_codebooks: usize,codebook_size: usize,@@ -600,24 +567,26 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {// Write first_code at position 0 from CPU.{- let token_indices = &mut semantic_bridge.codebook_token_indices;- if token_indices.shape() != [num_codebooks] || token_indices.data_type() != DataType::U32 {+ if semantic_bridge.num_codebooks != num_codebooks {return Err(Error::GenerateFailed);}- let indices_slice = token_indices.as_slice_mut::<u32>();- indices_slice[0] = first_code;+ let buffer_range = semantic_bridge.codebook_token_indices.as_buffer_range_ref();+ let range = buffer_range.range();+ unsafe {+ let ptr = (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start) as *mut u32;+ *ptr = first_code;+ }}// Blit residual codes from GPU: async_chain_results[0..N-2] -> codebook_token_indices[1..N-1]let residual_count = num_codebooks.saturating_sub(1);if residual_count > 0 {let dst_byte_offset = std::mem::size_of::<u32>(); // skip slot 0- let token_indices_buffer = semantic_bridge.codebook_token_indices.buffer();- let mut token_indices_buffer = token_indices_buffer.borrow_mut();source_runner.copy_async_chain_results_to_on(encoder,source_slot,- (&mut *token_indices_buffer, semantic_bridge.codebook_token_indices.offset() + dst_byte_offset),+ &mut semantic_bridge.codebook_token_indices,+ dst_byte_offset,residual_count,)?;}@@ -630,25 +599,16 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {let codebook_token_indices = &semantic_bridge.codebook_token_indices;let codebook_embeddings = &semantic_bridge.codebook_embeddings;- let slow_sum = slow_sum_embedding;- if codebook_token_indices.shape() != [num_codebooks]- || codebook_embeddings.shape() != [total_vocab, slow_model_dim]- || slow_sum.shape() != [1, slow_model_dim]- || codebook_embeddings.data_type() != slow_sum.data_type()+ if semantic_bridge.num_codebooks != num_codebooks+ || semantic_bridge.codebook_size != codebook_size+ || semantic_bridge.slow_model_dim != slow_model_dim{return Err(Error::GenerateFailed);}-- let codebook_token_indices_buffer = codebook_token_indices.buffer();- let codebook_token_indices_buffer = codebook_token_indices_buffer.borrow();- let codebook_embeddings_buffer = codebook_embeddings.buffer();- let codebook_embeddings_buffer = codebook_embeddings_buffer.borrow();- let slow_sum_buffer = slow_sum.buffer();- let mut slow_sum_buffer = slow_sum_buffer.borrow_mut();semantic_bridge.embedding_rows_sum.encode(- (&*codebook_token_indices_buffer, codebook_token_indices.offset()),- (&*codebook_embeddings_buffer, codebook_embeddings.offset()),- (&mut *slow_sum_buffer, slow_sum.offset()),+ codebook_token_indices,+ codebook_embeddings,+ slow_sum_embedding,num_codebooks_u32,total_vocab_u32,slow_model_dim_u32,@@ -680,8 +640,19 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {let context = Rc::clone(self.fast_runner.context());let mut shared_encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;- let total_fast_count =- self.encode_fast_passes_on(&mut shared_encoder, first_code, fast_vocab_limit, followup_count, sampling)?;+ let mut pending_token_inputs = Vec::new();+ let mut pending_sampling_inputs = Vec::new();+ let mut pending_sampling_outputs = Vec::new();+ let total_fast_count = self.encode_fast_passes_on(+ &mut shared_encoder,+ first_code,+ fast_vocab_limit,+ followup_count,+ sampling,+ &mut pending_token_inputs,+ &mut pending_sampling_inputs,+ &mut pending_sampling_outputs,+ )?;self.encode_slow_pass_on(&mut shared_encoder,current_semantic_token,@@ -689,8 +660,12 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {post_scale,sampling,by_codebook,+ &mut pending_token_inputs,+ &mut pending_sampling_inputs,+ &mut pending_sampling_outputs,)?;- self.slow_runner.submit_and_wait_command_buffer(shared_encoder)?;+ let submit_result = self.slow_runner.submit_and_wait_command_buffer(shared_encoder);+ submit_result?;for pass in 0..total_fast_count {let sampled = self.fast_runner.read_async_chain_result(pass)?;@@ -714,6 +689,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {fast_vocab_limit: usize,followup_count: usize,sampling: &mut TextSamplingState,+ pending_token_inputs: &mut Vec<TokenInputs<B>>,+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,+ pending_sampling_outputs: &mut Vec<Allocation<B>>,) -> Result<usize, Error> {let (slow_runner, fast_runner, semantic_bridge) =(&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);@@ -721,18 +699,16 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {let slow_model_dim = self.slow_model_dim;let fast_model_dim = self.fast_model_dim;- let mut pre_projection =- |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {- Self::encode_project_slow_hidden_to_fast_on(- runner.context().as_ref(),- semantic_bridge,- slow_hidden_capture,- &runner.single_override_embedding,- slow_model_dim,- fast_model_dim,- encoder,- )- };+ let mut pre_projection = |runner: &mut TokenDecoderRunner<B>, encoder: &mut Encoder<B>| {+ Self::encode_project_slow_hidden_to_fast_on(+ semantic_bridge,+ slow_hidden_capture,+ &mut runner.single_override_embedding,+ slow_model_dim,+ fast_model_dim,+ encoder,+ )+ };fast_runner.encode_first_and_followup_tokens_on(encoder,@@ -742,6 +718,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {followup_count,Some(fast_vocab_limit),sampling,+ pending_token_inputs,+ pending_sampling_inputs,+ pending_sampling_outputs,)}@@ -754,6 +733,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {post_scale: Option<f32>,sampling: &mut TextSamplingState,by_codebook: &[Vec<u32>],+ pending_token_inputs: &mut Vec<TokenInputs<B>>,+ pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,+ pending_sampling_outputs: &mut Vec<Allocation<B>>,) -> Result<(), Error> {let (slow_runner, fast_runner, semantic_bridge) =(&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);@@ -772,20 +754,19 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {None};- let mut pre_codebook_sum =- |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {- Self::encode_slow_codebook_sum_from_gpu_tokens_on(- semantic_bridge,- &runner.single_override_embedding,- first_code,- fast_runner,- 0,- num_codebooks,- codebook_size,- slow_model_dim,- encoder,- )- };+ let mut pre_codebook_sum = |runner: &mut TokenDecoderRunner<B>, encoder: &mut Encoder<B>| {+ Self::encode_slow_codebook_sum_from_gpu_tokens_on(+ semantic_bridge,+ &mut runner.single_override_embedding,+ first_code,+ fast_runner,+ 0,+ num_codebooks,+ codebook_size,+ slow_model_dim,+ encoder,+ )+ };slow_runner.encode_next_step_on(encoder,@@ -797,6 +778,9 @@ impl<B: Backend> FishAudioTextDecoderRuntime<B> {slow_sampling_mask,true, // capture_hiddenSome(&mut pre_codebook_sum),+ pending_token_inputs,+ pending_sampling_inputs,+ pending_sampling_outputs,)}diff --git a/crates/backend-uzu/src/speculators/ngram_speculator.rs b/crates/backend-uzu/src/speculators/ngram_speculator.rsindex 077e62174..cd8e5af60 100644--- a/crates/backend-uzu/src/speculators/ngram_speculator.rs+++ b/crates/backend-uzu/src/speculators/ngram_speculator.rs@@ -182,8 +182,7 @@ impl<B: Deref<Target = [u8]> + Send + Sync> NGramSpeculator<B> {let mut off = 0;let max_order = u32::from_le_bytes(bytes[off..off + 4].try_into().unwrap());- // Discount parameter is baked into the exported probabilities at training time.- let _discount = f32::from_le_bytes(bytes[off + 4..off + 8].try_into().unwrap());+ // Bytes [4..8] store the discount parameter, but exported probabilities are already discounted.off += 8;let mut tables: Vec<TaggedTableLayout> = Vec::with_capacity(max_order as usize);diff --git a/crates/backend-uzu/src/tracer/trace_validator.rs b/crates/backend-uzu/src/tracer/trace_validator.rsindex 2b06ba8cf..43f8a3bea 100644--- a/crates/backend-uzu/src/tracer/trace_validator.rs+++ b/crates/backend-uzu/src/tracer/trace_validator.rs@@ -16,18 +16,13 @@ use ndarray::{IxDyn, s};use num_traits::NumCast;use crate::{- ArrayElement, DataType,+ ArrayElement, DataType, allocation_to_vec,array::Array,- backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},+ backends::common::{Allocation, Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},classifier::Classifier,config::ModelMetadata,- encodable_block::{EncodingParameters, Sampling},- forward_pass::{- cache_layers::CacheLayers,- scratch_buffers::ScratchBuffers,- state::{ArrayId, ForwardPassState},- traces::ActivationTrace,- },+ encodable_block::{DecoderDecodeInput, EncodingParameters, Sampling},+ forward_pass::{cache_layers::CacheLayers, token_inputs::TokenInputs, traces::ActivationTrace},language_model::{language_model_generator_context::LanguageModelGeneratorContext,sampler::{ArgmaxSampler, LogitsSampler},@@ -196,7 +191,6 @@ impl<B: Backend> TraceValidator<B> {SpeculatorConfig::default(),SamplingSeed::default(),AsyncBatchSize::default(),- false,);let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &metadata)?;let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());@@ -246,39 +240,46 @@ impl<B: Backend> TraceValidator<B> {let token_ids = Self::load_array_as_vec::<i32, u64>(&traces_view, "activation_trace.token_ids");let token_positions = Self::load_array_as_vec::<i32, usize>(&traces_view, "activation_trace.token_positions");- let token_seeds: Vec<u64> = vec![0; token_ids.len()];-- let mut state = ForwardPassState::new_llm(- ctx.context.clone(),- &ctx.decoder_config,+ let token_inputs = TokenInputs::new_llm(+ ctx.context.as_ref(),&ctx.model_shape,- &ctx.scratch_buffers,- ctx.cache_layers.clone(),- ctx.shared_buffers.clone(),&token_ids,None,&token_positions,None,- &token_seeds,- token_ids.len(),+ None,/*sampling_start=*/ 0,/*sampling_length=*/ token_ids.len(),- false,- false,- None,- None,);+ let mut traces = ActivationTrace::new_llm(ctx.context.as_ref(), &ctx.model_shape, token_ids.len());let mut encoder =Encoder::<B>::new(ctx.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;-- ctx.executables- .encode(&mut state, &EncodingParameters::new(), &mut encoder)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ {+ let mut cache_layers = ctx.cache_layers.borrow_mut();+ let decoder_arguments = token_inputs.decoder_arguments(+ &ctx.model_shape,+ ctx.shared_buffers.as_ref(),+ Some(&mut *cache_layers),+ token_ids.len(),+ /*sampling_start=*/ 0,+ /*sampling_length=*/ token_ids.len(),+ #[cfg(feature = "tracing")]+ Some(&mut traces),+ );+ ctx.executables+ .encode_decode(+ decoder_arguments,+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),+ None,+ &EncodingParameters::new(),+ &mut encoder,+ )+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ }let pending = encoder.end_encoding().submit();pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;- let traces = state.traces().clone();let data_type = ctx.model_shape.activation_data_type();// Common layer validation@@ -286,38 +287,50 @@ impl<B: Backend> TraceValidator<B> {// LLM-specific: KV cache validationlet transformer_layers: Vec<usize> = {- let cache = state.cache_layers().unwrap().borrow();+ let cache = ctx.cache_layers.borrow();cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_transformer().map(|_| index)).collect()};for index in transformer_layers {- let keys = state.array(ArrayId::Keys(index));- let values = state.array(ArrayId::Values(index));+ let cache = ctx.cache_layers.borrow();+ let kv = cache.data[index].as_transformer().expect("Expected transformer layer");if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.keys", index),- metrics: Self::validate_array(data_type, &expected, &keys, Some(ArrayTransform::KVCacheSlice)),+ metrics: Self::validate_allocation(+ data_type,+ &expected,+ &kv.keys,+ &kv.shape,+ Some(ArrayTransform::KVCacheSlice),+ ),});}if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.values", index),- metrics: Self::validate_array(data_type, &expected, &values, Some(ArrayTransform::KVCacheSlice)),+ metrics: Self::validate_allocation(+ data_type,+ &expected,+ &kv.values,+ &kv.shape,+ Some(ArrayTransform::KVCacheSlice),+ ),});}}// LLM-specific: SSM state validationlet ssm_layers: Vec<usize> = {- let cache = state.cache_layers().unwrap().borrow();+ let cache = ctx.cache_layers.borrow();cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_state_space().map(|_| index)).collect()};for index in ssm_layers {- let conv_state = state.array(ArrayId::SsmConvState(index));- let ssm_state = state.array(ArrayId::SsmState(index));+ let cache = ctx.cache_layers.borrow();+ let ssm = cache.data[index].as_state_space().expect("Expected SSM layer");for path in [format!("updated_state.{}.conv_state", index),@@ -326,10 +339,11 @@ impl<B: Backend> TraceValidator<B> {if let Ok(expected) = traces_view.leaf_array(&path) {results.push(TracerValidationResult {name: path,- metrics: Self::validate_array(+ metrics: Self::validate_optional_allocation(data_type,&expected,- &conv_state,+ ssm.conv_state.as_ref(),+ &ssm.conv_shape,Some(ArrayTransform::SsmConvState),),});@@ -343,7 +357,7 @@ impl<B: Backend> TraceValidator<B> {if let Ok(expected) = traces_view.leaf_array(&path) {results.push(TracerValidationResult {name: path,- metrics: Self::validate_array(data_type, &expected, &ssm_state, None),+ metrics: Self::validate_allocation(data_type, &expected, &ssm.ssm_state, &ssm.ssm_shape, None),});}}@@ -351,13 +365,13 @@ impl<B: Backend> TraceValidator<B> {// LLM-specific: DeltaNet state validationlet delta_net_layers: Vec<usize> = {- let cache = state.cache_layers().unwrap().borrow();+ let cache = ctx.cache_layers.borrow();cache.data.iter().enumerate().filter_map(|(index, layer)| layer.as_delta_net().map(|_| index)).collect()};for index in delta_net_layers {- let conv_state = state.array(ArrayId::DeltaNetConvState(index));- let ssm_state = state.array(ArrayId::DeltaNetSsmState(index));+ let cache = ctx.cache_layers.borrow();+ let delta = cache.data[index].as_delta_net().expect("Expected DeltaNet layer");for path in [format!("updated_state.{}.conv_state", index),@@ -366,10 +380,11 @@ impl<B: Backend> TraceValidator<B> {if let Ok(expected) = traces_view.leaf_array(&path) {results.push(TracerValidationResult {name: path,- metrics: Self::validate_array(+ metrics: Self::validate_allocation(data_type,&expected,- &conv_state,+ &delta.conv_state,+ &delta.conv_shape,Some(ArrayTransform::SsmConvState),),});@@ -383,7 +398,13 @@ impl<B: Backend> TraceValidator<B> {if let Ok(expected) = traces_view.leaf_array(&path) {results.push(TracerValidationResult {name: path,- metrics: Self::validate_array(data_type, &expected, &ssm_state, None),+ metrics: Self::validate_allocation(+ data_type,+ &expected,+ &delta.ssm_state,+ &delta.ssm_shape,+ None,+ ),});}}@@ -392,7 +413,7 @@ impl<B: Backend> TraceValidator<B> {// LLM-specific: Token comparisonlet tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {let expected_tokens = Self::get_tokens_from_logits(&expected_logits);- let produced_tokens = Self::get_tokens_from_logits(&traces.borrow().logits);+ let produced_tokens = Self::get_tokens_from_logits(&traces.logits);expected_tokens.iter().zip(produced_tokens.iter())@@ -506,68 +527,68 @@ impl<B: Backend> TraceValidator<B> {// ========================================================================fn validate_layer_traces(- traces: &Rc<RefCell<ActivationTrace<B>>>,+ traces: &ActivationTrace<B>,traces_view: &ParameterTree<B::Context>,data_type: DataType,) -> Vec<TracerValidationResult> {let mut results = Vec::new();let validate = |path: &str, array: &Array<B>| -> Option<TracerValidationResult> {- if traces_view.leaf_array(path).is_ok() {+ if let Ok(expected) = traces_view.leaf_array(path) {Some(TracerValidationResult {name: path.to_string(),- metrics: Self::validate_array_with_name(data_type, traces_view, path, array),+ metrics: Self::validate_allocation(data_type, &expected, array.allocation(), array.shape(), None),})} else {None}};- for (index, layer_traces) in traces.borrow().layer_results.iter().enumerate() {+ for (index, layer_traces) in traces.layer_results.iter().enumerate() {let path = |suffix: &str| -> String {format!("activation_trace.layer_results.{}.activation_trace.{}", index, suffix)};- if let Some(r) = validate(&path("inputs"), &layer_traces.borrow().inputs) {+ if let Some(r) = validate(&path("inputs"), &layer_traces.inputs) {results.push(r);}- if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.borrow().pre_attention_norm) {+ if let Some(r) = validate(&path("pre_mixer_norm"), &layer_traces.pre_attention_norm) {results.push(r);}- if let Some(r) = validate(&path("mixer"), &layer_traces.borrow().attention) {+ if let Some(r) = validate(&path("mixer"), &layer_traces.attention) {results.push(r);}- if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.borrow().post_attention_norm) {+ if let Some(r) = validate(&path("post_mixer_norm"), &layer_traces.post_attention_norm) {results.push(r);}- if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.borrow().mlp_inputs) {+ if let Some(r) = validate(&path("mlp_inputs"), &layer_traces.mlp_inputs) {results.push(r);}- if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.borrow().pre_mlp_norm) {+ if let Some(r) = validate(&path("pre_mlp_norm"), &layer_traces.pre_mlp_norm) {results.push(r);}- if let Some(r) = validate(&path("mlp"), &layer_traces.borrow().mlp) {+ if let Some(r) = validate(&path("mlp"), &layer_traces.mlp) {results.push(r);}- if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.borrow().post_mlp_norm) {+ if let Some(r) = validate(&path("post_mlp_norm"), &layer_traces.post_mlp_norm) {results.push(r);}let outputs_path = format!("activation_trace.layer_results.{}.outputs", index);- if let Some(r) = validate(&outputs_path, &layer_traces.borrow().outputs) {+ if let Some(r) = validate(&outputs_path, &layer_traces.outputs) {results.push(r);}}// Output norm (common to all models)- if let Some(r) = validate("activation_trace.output_norm", &traces.borrow().output_norm) {+ if let Some(r) = validate("activation_trace.output_norm", &traces.output_norm) {results.push(r);}// Logits (common to all models, but path may vary)- if let Some(r) = validate("activation_trace.logits", &traces.borrow().logits) {+ if let Some(r) = validate("activation_trace.logits", &traces.logits) {results.push(r);- } else if let Some(r) = validate("logits", &traces.borrow().logits) {+ } else if let Some(r) = validate("logits", &traces.logits) {results.push(r);}@@ -575,22 +596,23 @@ impl<B: Backend> TraceValidator<B> {}fn validate_classifier_traces(- traces: &Rc<RefCell<ActivationTrace<B>>>,+ traces: &ActivationTrace<B>,traces_view: &ParameterTree<B::Context>,data_type: DataType,) -> Vec<TracerValidationResult> {let mut results = Vec::new();// Output pooling (classifier-specific)- if let Some(output_pooling) = &traces.borrow().output_pooling {- if traces_view.leaf_array("activation_trace.output_pooling").is_ok() {+ if let Some(output_pooling) = &traces.output_pooling {+ if let Ok(expected) = traces_view.leaf_array("activation_trace.output_pooling") {results.push(TracerValidationResult {name: "activation_trace.output_pooling".to_string(),- metrics: Self::validate_array_with_name(+ metrics: Self::validate_allocation(data_type,- traces_view,- "activation_trace.output_pooling",- output_pooling,+ &expected,+ output_pooling.allocation(),+ output_pooling.shape(),+ None,),});}@@ -600,40 +622,78 @@ impl<B: Backend> TraceValidator<B> {}// ========================================================================- // Array Validation+ // Allocation Validation// ========================================================================- fn validate_array(+ fn validate_allocation(data_type: DataType,expected_array: &Array<B>,- produced_array: &Array<B>,+ produced_allocation: &Allocation<B>,+ produced_shape: &[usize],transform: Option<ArrayTransform>,) -> TracerValidationMetrics {match data_type {- DataType::F16 => Self::validate_array_of_type::<f16>(expected_array, produced_array, transform),- DataType::BF16 => Self::validate_array_of_type::<bf16>(expected_array, produced_array, transform),- DataType::F32 => Self::validate_array_of_type::<f32>(expected_array, produced_array, transform),+ DataType::F16 => {+ Self::validate_allocation_of_type::<f16>(expected_array, produced_allocation, produced_shape, transform)+ },+ DataType::BF16 => Self::validate_allocation_of_type::<bf16>(+ expected_array,+ produced_allocation,+ produced_shape,+ transform,+ ),+ DataType::F32 => {+ Self::validate_allocation_of_type::<f32>(expected_array, produced_allocation, produced_shape, transform)+ },_ => panic!("Unsupported data type: {:?}", data_type),}}- fn validate_array_with_name(+ fn validate_optional_allocation(data_type: DataType,- traces_view: &ParameterTree<B::Context>,- expected_array_path: &str,- produced_array: &Array<B>,+ expected_array: &Array<B>,+ produced_allocation: Option<&Allocation<B>>,+ produced_shape: &[usize],+ transform: Option<ArrayTransform>,+ ) -> TracerValidationMetrics {+ match produced_allocation {+ Some(produced_allocation) => {+ Self::validate_allocation(data_type, expected_array, produced_allocation, produced_shape, transform)+ },+ None => match data_type {+ DataType::F16 => {+ Self::validate_allocation_data_of_type::<f16>(expected_array, &[], produced_shape, transform)+ },+ DataType::BF16 => {+ Self::validate_allocation_data_of_type::<bf16>(expected_array, &[], produced_shape, transform)+ },+ DataType::F32 => {+ Self::validate_allocation_data_of_type::<f32>(expected_array, &[], produced_shape, transform)+ },+ _ => panic!("Unsupported data type: {:?}", data_type),+ },+ }+ }++ fn validate_allocation_of_type<Precision: ArrayElement>(+ expected_array: &Array<B>,+ produced_allocation: &Allocation<B>,+ produced_shape: &[usize],+ transform: Option<ArrayTransform>,) -> TracerValidationMetrics {- let expected_array = traces_view.leaf_array(expected_array_path).unwrap();- Self::validate_array(data_type, &expected_array, produced_array, None)+ let produced = allocation_to_vec::<B, Precision>(produced_allocation);+ Self::validate_allocation_data_of_type(expected_array, &produced, produced_shape, transform)}- fn validate_array_of_type<Precision: ArrayElement>(+ fn validate_allocation_data_of_type<Precision: ArrayElement>(expected_array: &Array<B>,- produced_array: &Array<B>,+ produced_slice: &[Precision],+ produced_shape: &[usize],transform: Option<ArrayTransform>,) -> TracerValidationMetrics {let expected_view = expected_array.as_view::<Precision>();- let produced_view = produced_array.as_view::<Precision>();+ let produced_view = ndarray::ArrayView::from_shape(IxDyn(produced_shape), produced_slice)+ .expect("Failed to reshape allocation");let (mut expected_data, mut produced_data) = match transform {Some(ArrayTransform::KVCacheSlice) => {@@ -673,7 +733,6 @@ impl<B: Backend> TraceValidator<B> {let expected_shape = expected_data.shape().to_vec();let produced_shape = produced_data.shape().to_vec();- // Handle shape mismatches with leading dimension of 1if expected_shape != produced_shape {if expected_shape.len() == produced_shape.len() + 1&& expected_shape.get(0) == Some(&1)@@ -699,7 +758,6 @@ impl<B: Backend> TraceValidator<B> {}let reference: Vec<f32> = expected_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();-let result: Vec<f32> = produced_data.iter().map(|value| NumCast::from(*value).unwrap_or(0.0)).collect();let (atol, rtol, allowed_voilations_tol) = match expected_array.data_type() {@@ -728,6 +786,28 @@ impl<B: Backend> TraceValidator<B> {fraction_of_allowed_violations: f32,) -> TracerValidationMetrics {assert_eq!(result.len(), reference.len());+ if reference.is_empty() {+ return TracerValidationMetrics {+ atol,+ rtol,+ fraction_of_allowed_violations,+ reference_shape,+ result_shape,+ num_violations: 0,+ max_allowed_violations: 0,+ max_err_idx: 0,+ max_err: 0.0,+ max_err_rel: 0.0,+ max_err_reference_value: 0.0,+ rms_diff: 0.0,+ rms_result: 0.0,+ rms_reference: 0.0,+ rel_rms_reference: 0.0,+ diff_max: 0.0,+ diff_avg: 0.0,+ result_nan: false,+ };+ }let mut num_violations = 0;let mut max_err = 0.0f32;@@ -848,10 +928,6 @@ impl<B: Backend> TraceValidator<B> {return;}- let decoder_config = &context.decoder_config;- context.scratch_buffers =- ScratchBuffers::new(context.context.as_ref(), decoder_config, &context.model_shape, desired_suffix_length);-context.cache_layers = Rc::new(RefCell::new(CacheLayers::new(context.context.as_ref(),&context.model_shape,@@ -859,6 +935,7 @@ impl<B: Backend> TraceValidator<B> {desired_suffix_length,)));+ let decoder_config = context.model_config.decoder_config().expect("Failed to resolve decoder config");let intermediate_dtype: DataType = decoder_config.output_norm_config.scale_precision.into();context.kv_cache_update = Box::new(@@ -866,13 +943,8 @@ impl<B: Backend> TraceValidator<B> {.expect("Failed to create KV cache update kernel"),);- context.gpu_sampler = Sampling::new(- context.context.as_ref(),- intermediate_dtype,- desired_suffix_length,- decoder_config.vocab_size,- )- .expect("Failed to create sampling kernel");+ context.gpu_sampler =+ Sampling::new(context.context.as_ref(), intermediate_dtype).expect("Failed to create sampling kernel");}fn get_tokens_from_logits(logits: &Array<B>) -> Vec<u64> {diff --git a/crates/backend-uzu/tests/common/helpers.rs b/crates/backend-uzu/tests/common/helpers.rsindex 8c3a79ccd..e30b9a93a 100644--- a/crates/backend-uzu/tests/common/helpers.rs+++ b/crates/backend-uzu/tests/common/helpers.rs@@ -1,32 +1,58 @@-use std::rc::Rc;+use std::{mem::size_of, rc::Rc};-use backend_uzu::backends::common::{Backend, Buffer, Context, DenseBuffer};+use backend_uzu::{+ ArrayElement, allocation_copy_from_slice,+ backends::common::{Allocation, AllocationType, Backend, Context, Encoder},+};-#[allow(dead_code)]-pub fn alloc_buffer<B: Backend, T>(+pub fn allocation_size_bytes<T>(elements_count: usize) -> usize {+ elements_count * size_of::<T>()+}++pub fn alloc_allocation<B: Backend, T>(context: &B::Context,elements_count: usize,-) -> B::DenseBuffer {- context.create_buffer(elements_count * size_of::<T>()).expect("Failed to create buffer")+) -> Allocation<B> {+ context+ .create_allocation(allocation_size_bytes::<T>(elements_count), AllocationType::Global)+ .expect("Failed to create allocation")}-pub fn alloc_buffer_with_data<B: Backend, T: bytemuck::NoUninit>(+pub fn alloc_allocation_with_data<B: Backend, T: ArrayElement>(context: &B::Context,data: &[T],-) -> B::DenseBuffer {- if data.len() == 0 {- // Metal doesn't allow creating 0-byte buffers, create a minimal buffer instead- return context.create_buffer(1).expect("Failed to create buffer");- }+) -> Allocation<B> {+ let mut allocation = context+ .create_allocation(allocation_size_bytes::<T>(data.len()), AllocationType::Global)+ .expect("Failed to create allocation");+ allocation_copy_from_slice(&mut allocation, data).expect("Failed to initialize allocation");+ allocation+}- let slice: &[u8] = bytemuck::cast_slice(data);- let buffer = context.create_buffer(slice.len()).expect("Failed to create buffer");- let bytes = unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, buffer.size()) };- bytes.copy_from_slice(slice);- buffer+pub fn allocation_to_vec<B: Backend, T: ArrayElement>(allocation: &Allocation<B>) -> Vec<T> {+ backend_uzu::allocation_to_vec(allocation)+}++pub fn allocation_prefix_to_vec<B: Backend, T: ArrayElement>(+ allocation: &Allocation<B>,+ elements_count: usize,+) -> Vec<T> {+ let mut values = allocation_to_vec::<B, T>(allocation);+ values.truncate(elements_count);+ values+}++pub fn write_allocation<B: Backend, T: ArrayElement>(+ allocation: &mut Allocation<B>,+ data: &[T],+) {+ allocation_copy_from_slice(allocation, data).expect("Failed to write allocation")}-#[allow(dead_code)]pub fn create_context<B: Backend>() -> Rc<<B as Backend>::Context> {B::Context::new().expect(format!("Failed to create context for {}", std::any::type_name::<B>()).as_str())}++pub fn submit_encoder<B: Backend>(encoder: Encoder<B>) {+ encoder.end_encoding().submit().wait_until_completed().unwrap();+}diff --git a/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rs b/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rsindex 75b90c04d..793085542 100644--- a/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rs+++ b/crates/backend-uzu/tests/integration/person_schema_metal_bitmask_test.rs@@ -1,9 +1,15 @@#![cfg(all(metal_backend, grammar_xgrammar))]-use std::{cell::RefCell, path::PathBuf, rc::Rc};+use std::path::PathBuf;-use backend_uzu::{Array, DataType, backends::metal::Metal};-use metal::{MTLDevice, MTLDeviceExt, MTLResourceOptions};+use backend_uzu::{+ Array, DataType,+ backends::{+ common::{AllocationType, Backend, Context},+ metal::Metal,+ },+};+use metal::MTLDevice;use schemars::JsonSchema;use serde::{Deserialize, Serialize};use tokenizers::Tokenizer;@@ -61,8 +67,10 @@ fn person_schema_metal_bitmask() {// xgrammar expects an int32 mask tensor of shape [buffer_size]let elems = batch * buffer_size;let bytes = elems * core::mem::size_of::<i32>();- let buffer = device.new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let metal_bitmask = unsafe { Array::<Metal>::from_parts(Rc::new(RefCell::new(buffer)), 0, &shape, DataType::I32) };+ let context = <Metal as Backend>::Context::new().expect("Metal context");+ let allocation =+ context.create_allocation(bytes.max(1), AllocationType::Global).expect("Failed to create allocation");+ let metal_bitmask = unsafe { Array::<Metal>::from_allocation(allocation, 0, &shape, DataType::I32) };let mut shape_i64 = [buffer_size as i64];let mut bitmask_tensor = DLTensor {diff --git a/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rsindex 8f5fcc270..1cbbd7302 100644--- a/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rs+++ b/crates/backend-uzu/tests/integration/session/chat_session/context_mode_test.rs@@ -26,7 +26,6 @@ fn build_decoding_config() -> DecodingConfig {SpeculatorConfig::default(),SamplingSeed::Custom(42),AsyncBatchSize::default(),- true,)}@@ -55,6 +54,25 @@ fn request(.to_string()}+fn is_unknown_answer(response: &str) -> bool {+ let response = response.to_lowercase();+ [+ "not sure",+ "not able",+ "not aware",+ "don't know",+ "do not know",+ "can't tell",+ "cannot tell",+ "no information",+ "any information",+ "more context",+ "more details",+ ]+ .iter()+ .any(|pattern| response.contains(pattern))+}+#[tag(heavy)]#[test]fn test_context_mode_none() {@@ -69,7 +87,8 @@ fn test_context_mode_none() {println!("response_1: {}", response_1);println!("response_2: {}", response_2);- assert_eq!(response_1, response_2);+ assert!(is_unknown_answer(&response_1));+ assert!(is_unknown_answer(&response_2));}#[tag(heavy)]diff --git a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rsindex f481101f7..94648abe6 100644--- a/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs+++ b/crates/backend-uzu/tests/integration/session/chat_session/text_session_test.rs@@ -7,7 +7,7 @@ use backend_uzu::{session::{Session,config::{DecodingConfig, RunConfig},- parameter::SamplingSeed,+ parameter::{AsyncBatchSize, SamplingSeed},types::{Input, Message, Output},},};@@ -31,6 +31,12 @@ fn test_text_session_base() {run(build_default_text(), build_decoding_config(), 128);}+#[tag(heavy)]+#[test]+fn test_text_session_base_async_batch_1() {+ run(build_default_text(), build_decoding_config().with_async_batch_size(AsyncBatchSize::Custom(1)), 128);+}+#[tag(heavy)]#[test]fn test_text_session_with_prompt_lookup_speculator() {diff --git a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rsindex 526a55681..9484154b5 100644--- a/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs+++ b/crates/backend-uzu/tests/performance/audio_kernel_perf_test.rs@@ -46,7 +46,7 @@ fn make_lengths(batch_size: usize,value: i32,) -> Array<Metal> {- context.create_array_from(&[batch_size], &vec![value; batch_size], "lengths")+ context.create_array_from(&[batch_size], &vec![value; batch_size])}// ==========================================================================@@ -121,21 +121,16 @@ fn audio_kernel_perf() {let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();- let tokens =- context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");+ let tokens = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32);let lengths = make_lengths(&context, batch_size, frames as i32);- let semantic_codebook = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");- let semantic_out_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");- let semantic_out_bias = context.create_array_zeros(&[INPUT_DIM], dt, "perf_sem_bias");- let residual_codebooks = context.create_array_zeros(- &[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM],- dt,- "perf_res_cbs",- );- let residual_out_proj =- context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");- let residual_out_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");- let output = context.create_array_zeros(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");+ let semantic_codebook = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt);+ let semantic_out_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt);+ let semantic_out_bias = context.create_array_zeros(&[INPUT_DIM], dt);+ let residual_codebooks =+ context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt);+ let residual_out_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt);+ let residual_out_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt);+ let mut output = context.create_array_zeros(&[batch_size * frames * INPUT_DIM], dt).into_allocation();let label = format!("QuantizerDecode [B={}, T={}, K={}, dim={}, cdim={}]",@@ -144,34 +139,25 @@ fn audio_kernel_perf() {let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let tok = tokens.buffer();- let tok = tok.borrow();- let len = lengths.buffer();- let len = len.borrow();- let scb = semantic_codebook.buffer();- let scb = scb.borrow();- let sp = semantic_out_proj.buffer();- let sp = sp.borrow();- let sb = semantic_out_bias.buffer();- let sb = sb.borrow();- let rcb = residual_codebooks.buffer();- let rcb = rcb.borrow();- let rp = residual_out_proj.buffer();- let rp = rp.borrow();- let rb = residual_out_bias.buffer();- let rb = rb.borrow();- let out = output.buffer();- let mut out = out.borrow_mut();+ let tok = tokens.allocation();+ let len = lengths.allocation();+ let scb = semantic_codebook.allocation();+ let sp = semantic_out_proj.allocation();+ let sb = semantic_out_bias.allocation();+ let rcb = residual_codebooks.allocation();+ let rp = residual_out_proj.allocation();+ let rb = residual_out_bias.allocation();+ let out = &mut output;kernel.encode(- &*tok,- &*len,- &*scb,- &*sp,- &*sb,- &*rcb,- &*rp,- &*rb,- &mut *out,+ tok,+ len,+ scb,+ sp,+ sb,+ rcb,+ rp,+ rb,+ out,batch_size as i32,TOTAL_CODEBOOKS as i32,frames as i32,@@ -208,32 +194,27 @@ fn audio_kernel_perf() {let seq_in = frames;let seq_out = seq_in * stride;let ksize = stride * 2;- let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");- let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");- let bias = context.create_array_zeros(&[cout], dt, "tconv_b");- let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt);+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt);+ let bias = context.create_array_zeros(&[cout], dt);+ let mut output = context.create_array_zeros(&[batch_size * cout * seq_out], dt).into_allocation();let lengths = make_lengths(&context, batch_size, seq_out as i32);let label = format!("TransConv {tag} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();kernel.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,cin as i32,cout as i32,seq_in as i32,@@ -261,32 +242,27 @@ fn audio_kernel_perf() {let seq_in = frames;let seq_out = seq_in * stride;let ksize = stride * 2;- let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");- let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");- let bias = context.create_array_zeros(&[cout], dt, "tconv_b");- let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt);+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt);+ let bias = context.create_array_zeros(&[cout], dt);+ let mut output = context.create_array_zeros(&[batch_size * cout * seq_out], dt).into_allocation();let lengths = make_lengths(&context, batch_size, seq_out as i32);let label = format!("TransConv dec{block_idx} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();kernel.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,cin as i32,cout as i32,seq_in as i32,@@ -321,32 +297,27 @@ fn audio_kernel_perf() {let cin = UP_CHANNELS[2]; // 256let cout = DECODER_DIM; // 1536let ksize = 7;- let input = context.create_array_zeros(&[batch_size * cin * fc_seq], dt, "fc_in");- let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");- let bias = context.create_array_zeros(&[cout], dt, "fc_b");- let output = context.create_array_zeros(&[batch_size * cout * fc_seq], dt, "fc_out");+ let input = context.create_array_zeros(&[batch_size * cin * fc_seq], dt);+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt);+ let bias = context.create_array_zeros(&[cout], dt);+ let mut output = context.create_array_zeros(&[batch_size * cout * fc_seq], dt).into_allocation();let lengths = make_lengths(&context, batch_size, fc_seq as i32);let label = format!("CausalConv1d first_conv [{cin}->{cout}, k={ksize}, T={fc_seq}]");let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();kernel.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,cin as i32,cout as i32,fc_seq as i32,@@ -372,31 +343,26 @@ fn audio_kernel_perf() {for &dilation in &[1, 3, 9] {let label =format!("CausalConv1d res dec{block_idx} [{ch}->{ch}, k={ksize}, d={dilation}, T={dec_frames}]");- let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");- let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");- let bias = context.create_array_zeros(&[ch], dt, "res_b");- let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt);+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt);+ let bias = context.create_array_zeros(&[ch], dt);+ let mut output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt).into_allocation();let lengths = make_lengths(&context, batch_size, dec_frames as i32);let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();kernel.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,ch as i32,ch as i32,dec_frames as i32,@@ -419,32 +385,27 @@ fn audio_kernel_perf() {let cout = 1;let ksize = 7;let final_frames = SEMANTIC_FRAMES_10S * 2 * 2 * 8 * 8 * 4 * 2; // full resolution- let input = context.create_array_zeros(&[batch_size * cin * final_frames], dt, "fc_in");- let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");- let bias = context.create_array_zeros(&[cout], dt, "fc_b");- let output = context.create_array_zeros(&[batch_size * cout * final_frames], dt, "fc_out");+ let input = context.create_array_zeros(&[batch_size * cin * final_frames], dt);+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt);+ let bias = context.create_array_zeros(&[cout], dt);+ let mut output = context.create_array_zeros(&[batch_size * cout * final_frames], dt).into_allocation();let lengths = make_lengths(&context, batch_size, final_frames as i32);let label = format!("CausalConv1d final_conv [{cin}->{cout}, k={ksize}, T={final_frames}]");let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();kernel.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,cin as i32,cout as i32,final_frames as i32,@@ -477,35 +438,29 @@ fn audio_kernel_perf() {let ksize = 7;let label = format!("CausalConv1dResidual dec{block_idx} [{ch}->{ch}, k={ksize}, T={dec_frames}]");- let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");- let residual = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_res");- let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");- let bias = context.create_array_zeros(&[ch], dt, "res_b");- let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt);+ let residual = context.create_array_zeros(&[batch_size * ch * dec_frames], dt);+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt);+ let bias = context.create_array_zeros(&[ch], dt);+ let mut output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt).into_allocation();let lengths = make_lengths(&context, batch_size, dec_frames as i32);let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let r = residual.buffer();- let r = r.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let r = residual.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();kernel.encode(- &*i,- &*r,- &*w,- &*b,- &mut *o,- &*l,+ i,+ r,+ w,+ b,+ o,+ l,ch as i32,ch as i32,dec_frames as i32,@@ -536,31 +491,26 @@ fn audio_kernel_perf() {let ksize = 7;let label = format!("CausalConv1dGrouped dw up{block_idx} [{ch}, g={ch}, k={ksize}, T={frames}]");- let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_in");- let weight = context.create_array_zeros(&[ch * 1 * ksize], dt, "dw_w");- let bias = context.create_array_zeros(&[ch], dt, "dw_b");- let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_out");+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt);+ let weight = context.create_array_zeros(&[ch * 1 * ksize], dt);+ let bias = context.create_array_zeros(&[ch], dt);+ let mut output = context.create_array_zeros(&[batch_size * ch * frames], dt).into_allocation();let lengths = make_lengths(&context, batch_size, frames as i32);let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();kernel.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,ch as i32,ch as i32,frames as i32,@@ -592,23 +542,20 @@ fn audio_kernel_perf() {let ch_in = DEC_CHANNELS[block_idx];{let label = format!("HalfSnake pre-tconv dec{block_idx} [{ch_in}ch, T={dec_frames}]");- let input = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_in");- let alpha = context.create_array_zeros(&[ch_in], dt, "sn_a");- let output = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_out");+ let input = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt);+ let alpha = context.create_array_zeros(&[ch_in], dt);+ let mut output = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt).into_allocation();let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let a = alpha.buffer();- let a = a.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();+ let i = input.allocation();+ let a = alpha.allocation();+ let o = &mut output;kernel.encode(- &*i,- &*a,- &mut *o,+ i,+ a,+ o,ch_in as i32,dec_frames as i32,ch_in as i32,@@ -628,23 +575,20 @@ fn audio_kernel_perf() {let ch_out = DEC_CHANNELS[block_idx + 1];{let label = format!("HalfSnake res dec{block_idx} [{ch_out}ch, T={dec_frames}]");- let input = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_in");- let alpha = context.create_array_zeros(&[ch_out], dt, "sn_a");- let output = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_out");+ let input = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt);+ let alpha = context.create_array_zeros(&[ch_out], dt);+ let mut output = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt).into_allocation();let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let a = alpha.buffer();- let a = a.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();+ let i = input.allocation();+ let a = alpha.allocation();+ let o = &mut output;kernel.encode(- &*i,- &*a,- &mut *o,+ i,+ a,+ o,ch_out as i32,dec_frames as i32,ch_out as i32,@@ -677,31 +621,26 @@ fn audio_kernel_perf() {// Norm{let label = format!("NormNcs up{block_idx} [{ch}ch, T={frames}]");- let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_in");- let scales = context.create_array_zeros(&[ch], dt, "norm_s");- let bias = context.create_array_zeros(&[ch], dt, "norm_b");- let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_out");+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt);+ let scales = context.create_array_zeros(&[ch], dt);+ let bias = context.create_array_zeros(&[ch], dt);+ let mut output = context.create_array_zeros(&[batch_size * ch * frames], dt).into_allocation();let lengths = make_lengths(&context, batch_size, frames as i32);let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let s = scales.buffer();- let s = s.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let s = scales.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();k_norm.encode(- &*i,- &*s,- &*b,- &mut *o,- &*l,+ i,+ s,+ b,+ o,+ l,ch as i32,frames as i32,1e-6_f32,@@ -718,31 +657,26 @@ fn audio_kernel_perf() {// Pointwise conv1 (ch -> ch){let label = format!("Conv1d pw up{block_idx} [{ch}->{ch}, k=1, T={frames}]");- let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_in");- let weight = context.create_array_zeros(&[ch * ch], dt, "pw_w");- let bias = context.create_array_zeros(&[ch], dt, "pw_b");- let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_out");+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt);+ let weight = context.create_array_zeros(&[ch * ch], dt);+ let bias = context.create_array_zeros(&[ch], dt);+ let mut output = context.create_array_zeros(&[batch_size * ch * frames], dt).into_allocation();let lengths = make_lengths(&context, batch_size, frames as i32);let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let w = weight.buffer();- let w = w.borrow();- let b = bias.buffer();- let b = b.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- let l = lengths.buffer();- let l = l.borrow();+ let i = input.allocation();+ let w = weight.allocation();+ let b = bias.allocation();+ let o = &mut output;+ let l = lengths.allocation();k_conv1d.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,ch as i32,ch as i32,frames as i32,@@ -765,17 +699,15 @@ fn audio_kernel_perf() {{let n = ch * frames;let label = format!("GELU up{block_idx} [n={n}]");- let input = context.create_array_zeros(&[n], dt, "gelu_in");- let output = context.create_array_zeros(&[n], dt, "gelu_out");+ let input = context.create_array_zeros(&[n], dt);+ let mut output = context.create_array_zeros(&[n], dt).into_allocation();let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();{- let i = input.buffer();- let i = i.borrow();- let o = output.buffer();- let mut o = o.borrow_mut();- k_act.encode(Some(&*i), &mut *o, n as u32, ActivationType::GELU, &mut encoder);+ let i = input.allocation();+ let o = &mut output;+ k_act.encode(Some(i), o, n as u32, ActivationType::GELU, &mut encoder);}encoder.end_encoding().submit().wait_until_completed().unwrap();});@@ -813,15 +745,14 @@ fn audio_kernel_perf() {let frames = SEMANTIC_FRAMES_10S;// Quantizer arrays- let p_q_tok = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");+ let p_q_tok = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32);let p_q_len = make_lengths(&context, batch_size, frames as i32);- let p_q_sem_cb = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");- let p_q_sem_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");- let p_q_sem_bias = context.create_array_zeros(&[INPUT_DIM], dt, "p_sem_bias");- let p_q_res_cbs =- context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");- let p_q_res_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");- let p_q_res_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");+ let p_q_sem_cb = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt);+ let p_q_sem_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt);+ let p_q_sem_bias = context.create_array_zeros(&[INPUT_DIM], dt);+ let p_q_res_cbs = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt);+ let p_q_res_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt);+ let p_q_res_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt);// Compute frame counts at each stagelet f_up0 = frames * UP_STRIDES[0]; // 430@@ -834,42 +765,41 @@ fn audio_kernel_perf() {// Pipeline scratch: large enough for the biggest intermediate// Biggest is decoder_dim * f_up1 or ch * f_dec stageslet max_elems: usize = batch_size * DECODER_DIM * f_dec3.max(f_up1);- let scratch_a = context.create_array_zeros(&[max_elems], dt, "pipe_sa");- let scratch_b = context.create_array_zeros(&[max_elems], dt, "pipe_sb");- // Third scratch buffer for residual connections (avoids RefCell borrow conflict)- let scratch_r = context.create_array_zeros(&[max_elems], dt, "pipe_sr");+ let mut scratch_a = context.create_array_zeros(&[max_elems], dt).into_allocation();+ let mut scratch_b = context.create_array_zeros(&[max_elems], dt).into_allocation();+ let scratch_r = context.create_array_zeros(&[max_elems], dt).into_allocation();// Upsample block 0: 1024->512, stride=2- let u0_w = context.create_array_zeros(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");- let u0_b = context.create_array_zeros(&[UP_CHANNELS[1]], dt, "u0_b");+ let u0_w = context.create_array_zeros(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt);+ let u0_b = context.create_array_zeros(&[UP_CHANNELS[1]], dt);// ConvNeXt 0 (ch=512)let c0_ch = UP_CHANNELS[1];- let c0_dw_w = context.create_array_zeros(&[c0_ch * 1 * 7], dt, "c0_dw_w");- let c0_dw_b = context.create_array_zeros(&[c0_ch], dt, "c0_dw_b");- let c0_ns = context.create_array_zeros(&[c0_ch], dt, "c0_ns");- let c0_nb = context.create_array_zeros(&[c0_ch], dt, "c0_nb");- let c0_p1_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p1_w");- let c0_p1_b = context.create_array_zeros(&[c0_ch], dt, "c0_p1_b");- let c0_p2_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p2_w");- let c0_p2_b = context.create_array_zeros(&[c0_ch], dt, "c0_p2_b");+ let c0_dw_w = context.create_array_zeros(&[c0_ch * 1 * 7], dt);+ let c0_dw_b = context.create_array_zeros(&[c0_ch], dt);+ let c0_ns = context.create_array_zeros(&[c0_ch], dt);+ let c0_nb = context.create_array_zeros(&[c0_ch], dt);+ let c0_p1_w = context.create_array_zeros(&[c0_ch * c0_ch], dt);+ let c0_p1_b = context.create_array_zeros(&[c0_ch], dt);+ let c0_p2_w = context.create_array_zeros(&[c0_ch * c0_ch], dt);+ let c0_p2_b = context.create_array_zeros(&[c0_ch], dt);// Upsample block 1: 512->256, stride=2- let u1_w = context.create_array_zeros(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");- let u1_b = context.create_array_zeros(&[UP_CHANNELS[2]], dt, "u1_b");+ let u1_w = context.create_array_zeros(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt);+ let u1_b = context.create_array_zeros(&[UP_CHANNELS[2]], dt);// ConvNeXt 1 (ch=256)let c1_ch = UP_CHANNELS[2];- let c1_dw_w = context.create_array_zeros(&[c1_ch * 1 * 7], dt, "c1_dw_w");- let c1_dw_b = context.create_array_zeros(&[c1_ch], dt, "c1_dw_b");- let c1_ns = context.create_array_zeros(&[c1_ch], dt, "c1_ns");- let c1_nb = context.create_array_zeros(&[c1_ch], dt, "c1_nb");- let c1_p1_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p1_w");- let c1_p1_b = context.create_array_zeros(&[c1_ch], dt, "c1_p1_b");- let c1_p2_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p2_w");- let c1_p2_b = context.create_array_zeros(&[c1_ch], dt, "c1_p2_b");+ let c1_dw_w = context.create_array_zeros(&[c1_ch * 1 * 7], dt);+ let c1_dw_b = context.create_array_zeros(&[c1_ch], dt);+ let c1_ns = context.create_array_zeros(&[c1_ch], dt);+ let c1_nb = context.create_array_zeros(&[c1_ch], dt);+ let c1_p1_w = context.create_array_zeros(&[c1_ch * c1_ch], dt);+ let c1_p1_b = context.create_array_zeros(&[c1_ch], dt);+ let c1_p2_w = context.create_array_zeros(&[c1_ch * c1_ch], dt);+ let c1_p2_b = context.create_array_zeros(&[c1_ch], dt);// first_conv: 256->1536, k=7- let fc_w = context.create_array_zeros(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");- let fc_b = context.create_array_zeros(&[DECODER_DIM], dt, "fc_b");+ let fc_w = context.create_array_zeros(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt);+ let fc_b = context.create_array_zeros(&[DECODER_DIM], dt);// Decoder block weights for all 4 blocksstruct DecBlockWeights {@@ -892,27 +822,23 @@ fn audio_kernel_perf() {let stride = DEC_STRIDES[i];let ksize = stride * 2;DecBlockWeights {- snake_a: context.create_array_zeros(&[cin], dt, &format!("d{i}_sa")),- tconv_w: context.create_array_zeros(&[cin * cout * ksize], dt, &format!("d{i}_tw")),- tconv_b: context.create_array_zeros(&[cout], dt, &format!("d{i}_tb")),- ru_s1: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s1"))),- ru_c1_w: std::array::from_fn(|j| {- context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))- }),- ru_c1_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c1b"))),- ru_s2: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s2"))),- ru_c2_w: std::array::from_fn(|j| {- context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))- }),- ru_c2_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c2b"))),+ snake_a: context.create_array_zeros(&[cin], dt),+ tconv_w: context.create_array_zeros(&[cin * cout * ksize], dt),+ tconv_b: context.create_array_zeros(&[cout], dt),+ ru_s1: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),+ ru_c1_w: std::array::from_fn(|_| context.create_array_zeros(&[cout * cout * 7], dt)),+ ru_c1_b: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),+ ru_s2: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),+ ru_c2_w: std::array::from_fn(|_| context.create_array_zeros(&[cout * cout * 7], dt)),+ ru_c2_b: std::array::from_fn(|_| context.create_array_zeros(&[cout], dt)),}}).collect();// final snake + conv + tanh- let final_snake_a = context.create_array_zeros(&[DEC_CHANNELS[4]], dt, "final_sa");- let final_conv_w = context.create_array_zeros(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");- let final_conv_b = context.create_array_zeros(&[1], dt, "final_cb");+ let final_snake_a = context.create_array_zeros(&[DEC_CHANNELS[4]], dt);+ let final_conv_w = context.create_array_zeros(&[1 * DEC_CHANNELS[4] * 7], dt);+ let final_conv_b = context.create_array_zeros(&[1], dt);// Length arrayslet len_up0 = make_lengths(&context, batch_size, f_up0 as i32);@@ -927,38 +853,27 @@ fn audio_kernel_perf() {let ms = measure("Full pipeline (quant -> 2 up -> first_conv -> 4 dec -> final)", || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();- let sa = scratch_a.buffer();- let sb = scratch_b.buffer();// --- Quantizer: output [B, T, input_dim] (NSC) ---{- let tok = p_q_tok.buffer();- let tok = tok.borrow();- let len = p_q_len.buffer();- let len = len.borrow();- let scb = p_q_sem_cb.buffer();- let scb = scb.borrow();- let sp = p_q_sem_proj.buffer();- let sp = sp.borrow();- let sbi = p_q_sem_bias.buffer();- let sbi = sbi.borrow();- let rcb = p_q_res_cbs.buffer();- let rcb = rcb.borrow();- let rp = p_q_res_proj.buffer();- let rp = rp.borrow();- let rb = p_q_res_bias.buffer();- let rb = rb.borrow();- let mut out = sa.borrow_mut();+ let tok = p_q_tok.allocation();+ let len = p_q_len.allocation();+ let scb = p_q_sem_cb.allocation();+ let sp = p_q_sem_proj.allocation();+ let sbi = p_q_sem_bias.allocation();+ let rcb = p_q_res_cbs.allocation();+ let rp = p_q_res_proj.allocation();+ let rb = p_q_res_bias.allocation();k_quant.encode(- &*tok,- &*len,- &*scb,- &*sp,- &*sbi,- &*rcb,- &*rp,- &*rb,- &mut *out,+ tok,+ len,+ scb,+ sp,+ sbi,+ rcb,+ rp,+ rb,+ &mut scratch_a,batch_size as i32,TOTAL_CODEBOOKS as i32,frames as i32,@@ -973,20 +888,15 @@ fn audio_kernel_perf() {// --- Upsample block 0: 1024->512, stride=2, NSC input ---{- let i = sa.borrow();- let w = u0_w.buffer();- let w = w.borrow();- let b = u0_b.buffer();- let b = b.borrow();- let mut o = sb.borrow_mut();- let l = len_up0.buffer();- let l = l.borrow();+ let w = u0_w.allocation();+ let b = u0_b.allocation();+ let l = len_up0.allocation();k_tconv.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_a,+ w,+ b,+ &mut scratch_b,+ l,UP_CHANNELS[0] as i32,UP_CHANNELS[1] as i32,frames as i32,@@ -1001,20 +911,15 @@ fn audio_kernel_perf() {}// ConvNeXt 0{- let i = sb.borrow();- let w = c0_dw_w.buffer();- let w = w.borrow();- let b = c0_dw_b.buffer();- let b = b.borrow();- let mut o = sa.borrow_mut();- let l = len_up0.buffer();- let l = l.borrow();+ let w = c0_dw_w.allocation();+ let b = c0_dw_b.allocation();+ let l = len_up0.allocation();k_conv_grp.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_b,+ w,+ b,+ &mut scratch_a,+ l,c0_ch as i32,c0_ch as i32,f_up0 as i32,@@ -1027,20 +932,15 @@ fn audio_kernel_perf() {);}{- let i = sa.borrow();- let s = c0_ns.buffer();- let s = s.borrow();- let b = c0_nb.buffer();- let b = b.borrow();- let mut o = sb.borrow_mut();- let l = len_up0.buffer();- let l = l.borrow();+ let s = c0_ns.allocation();+ let b = c0_nb.allocation();+ let l = len_up0.allocation();k_norm.encode(- &*i,- &*s,- &*b,- &mut *o,- &*l,+ &scratch_a,+ s,+ b,+ &mut scratch_b,+ l,c0_ch as i32,f_up0 as i32,1e-6_f32,@@ -1050,20 +950,15 @@ fn audio_kernel_perf() {);}{- let i = sb.borrow();- let w = c0_p1_w.buffer();- let w = w.borrow();- let b = c0_p1_b.buffer();- let b = b.borrow();- let mut o = sa.borrow_mut();- let l = len_up0.buffer();- let l = l.borrow();+ let w = c0_p1_w.allocation();+ let b = c0_p1_b.allocation();+ let l = len_up0.allocation();k_conv1d.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_b,+ w,+ b,+ &mut scratch_a,+ l,c0_ch as i32,c0_ch as i32,f_up0 as i32,@@ -1078,25 +973,18 @@ fn audio_kernel_perf() {);}{- let i = sa.borrow();- let mut o = sb.borrow_mut();- k_act.encode(Some(&*i), &mut *o, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);+ k_act.encode(Some(&scratch_a), &mut scratch_b, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);}{- let i = sb.borrow();- let w = c0_p2_w.buffer();- let w = w.borrow();- let b = c0_p2_b.buffer();- let b = b.borrow();- let mut o = sa.borrow_mut();- let l = len_up0.buffer();- let l = l.borrow();+ let w = c0_p2_w.allocation();+ let b = c0_p2_b.allocation();+ let l = len_up0.allocation();k_conv1d.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_b,+ w,+ b,+ &mut scratch_a,+ l,c0_ch as i32,c0_ch as i32,f_up0 as i32,@@ -1111,27 +999,20 @@ fn audio_kernel_perf() {);}{- let a = sa.borrow();- let mut o = sb.borrow_mut();- k_add.encode(&*a, &*a, &mut *o, (c0_ch * f_up0) as i32, &mut encoder);+ k_add.encode(&scratch_a, &scratch_a, &mut scratch_b, (c0_ch * f_up0) as i32, &mut encoder);}// --- Upsample block 1: 512->256, stride=2, NCS input ---{- let i = sb.borrow();- let w = u1_w.buffer();- let w = w.borrow();- let b = u1_b.buffer();- let b = b.borrow();- let mut o = sa.borrow_mut();- let l = len_up1.buffer();- let l = l.borrow();+ let w = u1_w.allocation();+ let b = u1_b.allocation();+ let l = len_up1.allocation();k_tconv.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_b,+ w,+ b,+ &mut scratch_a,+ l,UP_CHANNELS[1] as i32,UP_CHANNELS[2] as i32,f_up0 as i32,@@ -1146,20 +1027,15 @@ fn audio_kernel_perf() {}// ConvNeXt 1{- let i = sa.borrow();- let w = c1_dw_w.buffer();- let w = w.borrow();- let b = c1_dw_b.buffer();- let b = b.borrow();- let mut o = sb.borrow_mut();- let l = len_up1.buffer();- let l = l.borrow();+ let w = c1_dw_w.allocation();+ let b = c1_dw_b.allocation();+ let l = len_up1.allocation();k_conv_grp.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_a,+ w,+ b,+ &mut scratch_b,+ l,c1_ch as i32,c1_ch as i32,f_up1 as i32,@@ -1172,20 +1048,15 @@ fn audio_kernel_perf() {);}{- let i = sb.borrow();- let s = c1_ns.buffer();- let s = s.borrow();- let b = c1_nb.buffer();- let b = b.borrow();- let mut o = sa.borrow_mut();- let l = len_up1.buffer();- let l = l.borrow();+ let s = c1_ns.allocation();+ let b = c1_nb.allocation();+ let l = len_up1.allocation();k_norm.encode(- &*i,- &*s,- &*b,- &mut *o,- &*l,+ &scratch_b,+ s,+ b,+ &mut scratch_a,+ l,c1_ch as i32,f_up1 as i32,1e-6_f32,@@ -1195,20 +1066,15 @@ fn audio_kernel_perf() {);}{- let i = sa.borrow();- let w = c1_p1_w.buffer();- let w = w.borrow();- let b = c1_p1_b.buffer();- let b = b.borrow();- let mut o = sb.borrow_mut();- let l = len_up1.buffer();- let l = l.borrow();+ let w = c1_p1_w.allocation();+ let b = c1_p1_b.allocation();+ let l = len_up1.allocation();k_conv1d.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_a,+ w,+ b,+ &mut scratch_b,+ l,c1_ch as i32,c1_ch as i32,f_up1 as i32,@@ -1223,25 +1089,18 @@ fn audio_kernel_perf() {);}{- let i = sb.borrow();- let mut o = sa.borrow_mut();- k_act.encode(Some(&*i), &mut *o, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);+ k_act.encode(Some(&scratch_b), &mut scratch_a, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);}{- let i = sa.borrow();- let w = c1_p2_w.buffer();- let w = w.borrow();- let b = c1_p2_b.buffer();- let b = b.borrow();- let mut o = sb.borrow_mut();- let l = len_up1.buffer();- let l = l.borrow();+ let w = c1_p2_w.allocation();+ let b = c1_p2_b.allocation();+ let l = len_up1.allocation();k_conv1d.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_a,+ w,+ b,+ &mut scratch_b,+ l,c1_ch as i32,c1_ch as i32,f_up1 as i32,@@ -1256,27 +1115,20 @@ fn audio_kernel_perf() {);}{- let a = sb.borrow();- let mut o = sa.borrow_mut();- k_add.encode(&*a, &*a, &mut *o, (c1_ch * f_up1) as i32, &mut encoder);+ k_add.encode(&scratch_b, &scratch_b, &mut scratch_a, (c1_ch * f_up1) as i32, &mut encoder);}// --- first_conv: 256->1536, k=7 ---{- let i = sa.borrow();- let w = fc_w.buffer();- let w = w.borrow();- let b = fc_b.buffer();- let b = b.borrow();- let mut o = sb.borrow_mut();- let l = len_up1.buffer();- let l = l.borrow();+ let w = fc_w.allocation();+ let b = fc_b.allocation();+ let l = len_up1.allocation();k_conv1d_causal.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ &scratch_a,+ w,+ b,+ &mut scratch_b,+ l,UP_CHANNELS[2] as i32,DECODER_DIM as i32,f_up1 as i32,@@ -1304,19 +1156,16 @@ fn audio_kernel_perf() {// snake (in-place swap){- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let a = blk.snake_a.buffer();- let a = a.borrow();- let mut o = o_buf.borrow_mut();+ let a = blk.snake_a.allocation();k_snake.encode(- &*i,- &*a,- &mut *o,+ i,+ a,+ o,cin as i32,cur_frames as i32,cin as i32,@@ -1330,25 +1179,20 @@ fn audio_kernel_perf() {// trans_conv{- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let w = blk.tconv_w.buffer();- let w = w.borrow();- let b = blk.tconv_b.buffer();- let b = b.borrow();- let mut o = o_buf.borrow_mut();- let l = dec_len.buffer();- let l = l.borrow();+ let w = blk.tconv_w.allocation();+ let b = blk.tconv_b.allocation();+ let l = dec_len.allocation();k_tconv.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,cin as i32,cout as i32,cur_frames as i32,@@ -1365,23 +1209,19 @@ fn audio_kernel_perf() {cur_frames = next_frames;// 3 residual units- let sr = scratch_r.buffer();for ru_idx in 0..3 {// snake1{- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let a = blk.ru_s1[ru_idx].buffer();- let a = a.borrow();- let mut o = o_buf.borrow_mut();+ let a = blk.ru_s1[ru_idx].allocation();k_snake.encode(- &*i,- &*a,- &mut *o,+ i,+ a,+ o,cout as i32,cur_frames as i32,cout as i32,@@ -1395,25 +1235,20 @@ fn audio_kernel_perf() {// conv1 (dilation varies: 1, 3, 9){- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let w = blk.ru_c1_w[ru_idx].buffer();- let w = w.borrow();- let b = blk.ru_c1_b[ru_idx].buffer();- let b = b.borrow();- let mut o = o_buf.borrow_mut();- let l = dec_len.buffer();- let l = l.borrow();+ let w = blk.ru_c1_w[ru_idx].allocation();+ let b = blk.ru_c1_b[ru_idx].allocation();+ let l = dec_len.allocation();k_conv1d_causal.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,cout as i32,cout as i32,cur_frames as i32,@@ -1428,19 +1263,16 @@ fn audio_kernel_perf() {// snake2{- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let a = blk.ru_s2[ru_idx].buffer();- let a = a.borrow();- let mut o = o_buf.borrow_mut();+ let a = blk.ru_s2[ru_idx].allocation();k_snake.encode(- &*i,- &*a,- &mut *o,+ i,+ a,+ o,cout as i32,cur_frames as i32,cout as i32,@@ -1453,30 +1285,23 @@ fn audio_kernel_perf() {cur_in_sa = !cur_in_sa;// conv2 + residual add- // Use scratch_r as the residual source (separate buffer to- // avoid RefCell borrow conflict with the output destination).{- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let r = sr.borrow();- let w = blk.ru_c2_w[ru_idx].buffer();- let w = w.borrow();- let b = blk.ru_c2_b[ru_idx].buffer();- let b = b.borrow();- let mut o = o_buf.borrow_mut();- let l = dec_len.buffer();- let l = l.borrow();+ let r = &scratch_r;+ let w = blk.ru_c2_w[ru_idx].allocation();+ let b = blk.ru_c2_b[ru_idx].allocation();+ let l = dec_len.allocation();k_conv_res.encode(- &*i,- &*r,- &*w,- &*b,- &mut *o,- &*l,+ i,+ r,+ w,+ b,+ o,+ l,cout as i32,cout as i32,cur_frames as i32,@@ -1493,19 +1318,16 @@ fn audio_kernel_perf() {// --- final snake + conv + tanh ---{- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let a = final_snake_a.buffer();- let a = a.borrow();- let mut o = o_buf.borrow_mut();+ let a = final_snake_a.allocation();k_snake.encode(- &*i,- &*a,- &mut *o,+ i,+ a,+ o,DEC_CHANNELS[4] as i32,f_dec3 as i32,DEC_CHANNELS[4] as i32,@@ -1517,25 +1339,20 @@ fn audio_kernel_perf() {}cur_in_sa = !cur_in_sa;{- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let w = final_conv_w.buffer();- let w = w.borrow();- let b = final_conv_b.buffer();- let b = b.borrow();- let mut o = o_buf.borrow_mut();- let l = len_dec3.buffer();- let l = l.borrow();+ let w = final_conv_w.allocation();+ let b = final_conv_b.allocation();+ let l = len_dec3.allocation();k_conv1d_causal.encode(- &*i,- &*w,- &*b,- &mut *o,- &*l,+ i,+ w,+ b,+ o,+ l,DEC_CHANNELS[4] as i32,1_i32,f_dec3 as i32,@@ -1549,14 +1366,12 @@ fn audio_kernel_perf() {cur_in_sa = !cur_in_sa;// tanh{- let (i_buf, o_buf) = if cur_in_sa {- (sa.clone(), sb.clone())+ let (i, o) = if cur_in_sa {+ (&scratch_a, &mut scratch_b)} else {- (sb.clone(), sa.clone())+ (&scratch_b, &mut scratch_a)};- let i = i_buf.borrow();- let mut o = o_buf.borrow_mut();- k_act.encode(Some(&*i), &mut *o, f_dec3 as u32, ActivationType::TANH, &mut encoder);+ k_act.encode(Some(i), o, f_dec3 as u32, ActivationType::TANH, &mut encoder);}encoder.end_encoding().submit().wait_until_completed().unwrap();diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rsindex 052b6e371..47a50d96e 100644--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_experts_perf_test.rs@@ -13,7 +13,7 @@ use backend_uzu::{use half::bf16;use rand::{RngExt, SeedableRng, rngs::StdRng};-use crate::common::helpers::{alloc_buffer, alloc_buffer_with_data, create_context};+use crate::common::helpers::{alloc_allocation, alloc_allocation_with_data, create_context};fn build_offsets(e: usize,@@ -77,99 +77,98 @@ fn run_decode_case<B: Backend>(let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(ctx).expect("experts decode kernel");- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x_perm);- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);let num_tiles_k = d_ff.div_ceil(K_TILE);- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);- let mut output_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);-// Buffers for indirect dispatch- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);let max_tiles = e * sum_k * 16384; // Conservative upper bound- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, sum_k);+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- experts_kernel.encode(- &mut encoder,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut output_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- data_type: DataType::BF16,- },- );+ let output = experts_kernel+ .encode(+ &mut encoder,+ MoeExpertsTwoPassArguments {+ x_perm: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &mut row_expert_map_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &mut tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ tile_map: &mut tile_map_buf,+ total_tiles: &mut total_tiles_buf,+ dispatch_args: &mut dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.702,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- experts_kernel.encode(- &mut encoder,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut output_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- data_type: DataType::BF16,- },- );+ let output = experts_kernel+ .encode(+ &mut encoder,+ MoeExpertsTwoPassArguments {+ x_perm: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &mut row_expert_map_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &mut tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ tile_map: &mut tile_map_buf,+ total_tiles: &mut total_tiles_buf,+ dispatch_args: &mut dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.702,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -227,34 +226,29 @@ fn run_two_pass_prefill_case<B: Backend>(let experts_kernel = MoeExpertsTwoPassPrefillBlock::<B>::new(ctx).expect("experts prefill kernel");- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x_perm);- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);let num_tiles_k = d_ff.div_ceil(K_TILE);- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);- let mut output_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);-- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);let max_tiles = e * sum_k * 16384;- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, sum_k);+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,+ x_perm: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut output_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,@@ -277,8 +271,9 @@ fn run_two_pass_prefill_case<B: Backend>(silu_alpha: 1.702,data_type: DataType::BF16,};- experts_kernel.encode(&mut encoder, args);+ let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);}let mut times = Vec::with_capacity(iters);@@ -286,11 +281,9 @@ fn run_two_pass_prefill_case<B: Backend>(let start = Instant::now();let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,+ x_perm: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut output_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,@@ -313,8 +306,9 @@ fn run_two_pass_prefill_case<B: Backend>(silu_alpha: 1.702,data_type: DataType::BF16,};- experts_kernel.encode(&mut encoder, args);+ let output = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -362,74 +356,73 @@ fn run_fused_single_token_case<B: Backend>(let fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(ctx).expect("fused kernel");- let x_buf = alloc_buffer_with_data::<B, bf16>(ctx, &x);- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(ctx, &topk_ids);- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(ctx, &topk_probs);- let w13_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w13_all);- let w2_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w2_all);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &down_biases);- let mut hidden_buf = alloc_buffer::<B, f32>(ctx, k * d_ff);- let mut y_buf = alloc_buffer::<B, bf16>(ctx, d_model);-+ let x_buf = alloc_allocation_with_data::<B, bf16>(ctx, &x);+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(ctx, &topk_ids);+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(ctx, &topk_probs);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w13_all);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w2_all);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- fused_kernel.encode(- &mut encoder,- MoeExpertsSingleDecodeArguments {- x: &x_buf,- topk_ids: &topk_ids_buf,- topk_probs: &topk_probs_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- hidden: &mut hidden_buf,- y: &mut y_buf,- d_model,- d_ff,- k,- gating_code: 2, // SwiGLU- silu_alpha: 1.0,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- data_type: DataType::BF16,- },- );+ let output = fused_kernel+ .encode(+ &mut encoder,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ d_model,+ d_ff,+ k,+ gating_code: 2, // SwiGLU+ silu_alpha: 1.0,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- fused_kernel.encode(- &mut encoder,- MoeExpertsSingleDecodeArguments {- x: &x_buf,- topk_ids: &topk_ids_buf,- topk_probs: &topk_probs_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- hidden: &mut hidden_buf,- y: &mut y_buf,- d_model,- d_ff,- k,- gating_code: 2, // SwiGLU- silu_alpha: 1.0,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- data_type: DataType::BF16,- },- );+ let output = fused_kernel+ .encode(+ &mut encoder,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ d_model,+ d_ff,+ k,+ gating_code: 2, // SwiGLU+ silu_alpha: 1.0,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -484,97 +477,96 @@ fn run_indirect_decode_timed<B: Backend>(let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(ctx).expect("decode kernel");- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(ctx, &x_perm);- let offsets_buf = alloc_buffer_with_data::<B, u32>(ctx, &offsets);- let w13_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w13);- let w2_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w2);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &down_biases);+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(ctx, &x_perm);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(ctx, &offsets);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w13);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w2);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);let num_tiles_k = d_ff.div_ceil(K_TILE);- let mut hidden_buf = alloc_buffer::<B, f32>(ctx, sum_k * d_ff);- let mut output_buf = alloc_buffer::<B, bf16>(ctx, sum_k * d_model);-- let mut tile_counts_buf = alloc_buffer::<B, u32>(ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(ctx, e + 1);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(ctx, e + 1);let max_tiles = e * sum_k * 16384;- let mut tile_map_buf = alloc_buffer::<B, u32>(ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_buffer::<B, u32>(ctx, 1);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(ctx, 3);- let mut row_expert_map_buf = alloc_buffer::<B, u32>(ctx, sum_k);+ let mut tile_map_buf = alloc_allocation::<B, u32>(ctx, max_tiles * 3);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(ctx, 1);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(ctx, 3);+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(ctx, sum_k);for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- experts_kernel.encode(- &mut encoder,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut output_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.0,- data_type: DataType::BF16,- },- );+ let output = experts_kernel+ .encode(+ &mut encoder,+ MoeExpertsTwoPassArguments {+ x_perm: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &mut row_expert_map_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &mut tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ tile_map: &mut tile_map_buf,+ total_tiles: &mut total_tiles_buf,+ dispatch_args: &mut dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.0,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- experts_kernel.encode(- &mut encoder,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut output_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.0,- data_type: DataType::BF16,- },- );+ let output = experts_kernel+ .encode(+ &mut encoder,+ MoeExpertsTwoPassArguments {+ x_perm: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &mut row_expert_map_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &mut tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ tile_map: &mut tile_map_buf,+ total_tiles: &mut total_tiles_buf,+ dispatch_args: &mut dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.0,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -610,74 +602,73 @@ fn run_fused_decode_timed<B: Backend>(let fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(ctx).expect("fused kernel");- let x_buf = alloc_buffer_with_data::<B, bf16>(ctx, &x);- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(ctx, &topk_ids);- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(ctx, &topk_probs);- let w13_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w13_all);- let w2_buf = alloc_buffer_with_data::<B, bf16>(ctx, &w2_all);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(ctx, &down_biases);- let mut hidden_buf = alloc_buffer::<B, f32>(ctx, k * d_ff);- let mut y_buf = alloc_buffer::<B, bf16>(ctx, d_model);-+ let x_buf = alloc_allocation_with_data::<B, bf16>(ctx, &x);+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(ctx, &topk_ids);+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(ctx, &topk_probs);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w13_all);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(ctx, &w2_all);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(ctx, &down_biases);for _ in 0..warmup {let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- fused_kernel.encode(- &mut encoder,- MoeExpertsSingleDecodeArguments {- x: &x_buf,- topk_ids: &topk_ids_buf,- topk_probs: &topk_probs_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- hidden: &mut hidden_buf,- y: &mut y_buf,- d_model,- d_ff,- k,- gating_code: 2,- silu_alpha: 1.0,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- data_type: DataType::BF16,- },- );+ let output = fused_kernel+ .encode(+ &mut encoder,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ d_model,+ d_ff,+ k,+ gating_code: 2,+ silu_alpha: 1.0,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");- fused_kernel.encode(- &mut encoder,- MoeExpertsSingleDecodeArguments {- x: &x_buf,- topk_ids: &topk_ids_buf,- topk_probs: &topk_probs_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- hidden: &mut hidden_buf,- y: &mut y_buf,- d_model,- d_ff,- k,- gating_code: 2,- silu_alpha: 1.0,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- data_type: DataType::BF16,- },- );+ let output = fused_kernel+ .encode(+ &mut encoder,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ d_model,+ d_ff,+ k,+ gating_code: 2,+ silu_alpha: 1.0,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);times.push(start.elapsed().as_secs_f64() * 1000.0);}diff --git a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rsindex af7db6b20..355dec81a 100644--- a/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs+++ b/crates/backend-uzu/tests/performance/encodable_block/moe/moe_perf_test.rs@@ -13,7 +13,7 @@ use half::bf16;use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::common::{- helpers::{alloc_buffer, alloc_buffer_with_data, create_context},+ helpers::{alloc_allocation, alloc_allocation_with_data, create_context},perf::run_perf_with_warmup,};@@ -39,11 +39,11 @@ fn test_moe_e2e_decode_perf() {let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();// Buffers (simplified - just key buffers for timing)- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_w);- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_b);- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_w);+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_b);+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk fused kernel");@@ -96,11 +96,11 @@ fn test_moe_e2e_prefill_perf() {let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();// Buffers (simplified - just key buffers for timing)- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_w);- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_b);- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_w);+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_b);+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);let router_topk = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk fused kernel");@@ -151,21 +151,22 @@ fn test_moe_pipeline_breakdown_decode() {let router_w: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.5..0.5))).collect();let router_b: Vec<bf16> = (0..e).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_w);- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_b);- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_w);+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_b);+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);let num_blocks = ((t + 255) / 256).max(1);let num_tiles = ((e + 512 - 1) / 512).max(1);- let mut partials_buf = alloc_buffer::<B, i32>(&ctx, num_blocks * num_tiles * e);- let mut offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut sumk_buf = alloc_buffer::<B, u32>(&ctx, 1);- let mut bucketed_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);- let mut x_perm_buf = alloc_buffer::<B, bf16>(&ctx, t * k * d_model);- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, t * k * d_model);- let mut tok2row_buf = alloc_buffer::<B, i32>(&ctx, t * k);- let mut y_out_buf = alloc_buffer::<B, bf16>(&ctx, t * d_model);+ let mut partials_buf = alloc_allocation::<B, i32>(&ctx, num_blocks * num_tiles * e);+ let mut offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut sumk_buf = alloc_allocation::<B, u32>(&ctx, 1);+ let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);+ let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, t * k * d_model);+ let y_partial: Vec<bf16> = (0..t * k * d_model).map(|_| bf16::from_f32(rng.random_range(-1.0..1.0))).collect();+ let y_partial_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &y_partial);+ let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);+ let mut y_out_buf = alloc_allocation::<B, bf16>(&ctx, t * d_model);// Expert weights buffers// Generate W13 in original layout [E, d_model, 2*d_ff]@@ -190,30 +191,28 @@ fn test_moe_pipeline_breakdown_decode() {let up_biases: Vec<bf16> = (0..e * 2 * d_ff).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);// Experts tiling buffers for two-passconst K_TILE: usize = 64;let sum_k = t * k;let num_tiles_k = (d_ff + K_TILE - 1) / K_TILE;let max_tiles = t * k * e * num_tiles_k;- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);- // Two-pass specific buffers- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, sum_k);+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, sum_k);// Scatter block bases buffers- let mut block_bases_buf = alloc_buffer::<B, u32>(&ctx, num_blocks * num_tiles);- let mut block_alloc_buf = alloc_buffer::<B, u32>(&ctx, num_blocks * num_tiles);- let mut bucketed_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);+ let mut block_bases_buf = alloc_allocation::<B, u32>(&ctx, num_blocks * num_tiles);+ let mut block_alloc_buf = alloc_allocation::<B, u32>(&ctx, num_blocks * num_tiles);+ let mut bucketed_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);// Create kernel structs (use production-validated encoding logic)let counts_offsets_kernel =@@ -302,10 +301,10 @@ fn test_moe_pipeline_breakdown_decode() {&mut encoder,DataType::BF16,MoeGatherArguments {- x_buffer: &x_buf,- bucketed_ids_buffer: &bucketed_ids_buf,- x_perm_buffer: &mut x_perm_buf,- sumk_buffer: &sumk_buf,+ x: &x_buf,+ bucketed_ids: &bucketed_ids_buf,+ x_perm: &mut x_perm_buf,+ sumk: &sumk_buf,t,k,d_model,@@ -316,38 +315,39 @@ fn test_moe_pipeline_breakdown_decode() {let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- experts_kernel.encode(- &mut encoder,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut y_partial_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2, // SILU- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: 20.0,- up_clip_min: -19.0,- up_clip_max: 21.0,- silu_alpha: 1.702,- data_type: DataType::BF16,- },- );+ let output = experts_kernel+ .encode(+ &mut encoder,+ MoeExpertsTwoPassArguments {+ x_perm: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &mut row_expert_map_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &mut tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ tile_map: &mut tile_map_buf,+ total_tiles: &mut total_tiles_buf,+ dispatch_args: &mut dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2, // SILU+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: 20.0,+ up_clip_min: -19.0,+ up_clip_max: 21.0,+ silu_alpha: 1.702,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");encoder.end_encoding().submit().wait_until_completed().unwrap();+ drop(output);});let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {diff --git a/crates/backend-uzu/tests/performance/matmul/bench.rs b/crates/backend-uzu/tests/performance/matmul/bench.rsindex c350907dc..d21ef12ae 100644--- a/crates/backend-uzu/tests/performance/matmul/bench.rs+++ b/crates/backend-uzu/tests/performance/matmul/bench.rs@@ -4,7 +4,7 @@ use backend_uzu::{DataType,backends::{common::{- Backend, Encoder,+ Allocation, AllocationType, AsBufferRangeMut, Backend, Context, DenseBuffer, Encoder,kernel::{ManualKernels,matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},@@ -13,23 +13,26 @@ use backend_uzu::{metal::Metal,},};-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use objc2::{rc::Retained, runtime::ProtocolObject};use super::{error::BenchError, output::PerfResult, shapes::TestShape};type Ctx = <Metal as Backend>::Context;-type Buf = Retained<ProtocolObject<dyn MTLBuffer>>;const WARMUP_ITERATIONS: usize = 3;const BENCHMARK_ITERATIONS: usize = 10;fn fill_buffer_random(- buffer: &ProtocolObject<dyn MTLBuffer>,+ allocation: &mut Allocation<Metal>,byte_count: usize,) {- let pointer = buffer.contents().as_ptr() as *mut u8;- let slice = unsafe { std::slice::from_raw_parts_mut(pointer, byte_count) };+ let buffer_range = allocation.as_buffer_range_mut();+ let range = buffer_range.range();+ let slice = unsafe {+ std::slice::from_raw_parts_mut(+ (buffer_range.buffer().cpu_ptr().as_ptr() as *mut u8).add(range.start),+ byte_count,+ )+ };for (i, byte) in slice.iter_mut().enumerate() {*byte = (i % 251) as u8;}@@ -39,21 +42,20 @@ fn encode_and_run(context: &Ctx,kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,shape: &TestShape,- a_buffer: &Buf,- b_buffer: &Buf,- d_buffer: &mut Buf,+ a_allocation: &backend_uzu::backends::common::Allocation<Metal>,+ b_allocation: &backend_uzu::backends::common::Allocation<Metal>,+ d_allocation: &mut backend_uzu::backends::common::Allocation<Metal>,) -> Result<f64, BenchError> {let mut encoder = Encoder::new(context).map_err(|_| BenchError::CommandBuffer)?;kernel.encode(- context,MatmulArguments {- a: a_buffer,+ a: a_allocation,a_offset: 0,- b: b_buffer,+ b: b_allocation,ab_scale: 1.0,c: MatmulArgumentC::None,- d: d_buffer,+ d: d_allocation,batch_dim: shape.batch as u32,input_dim: shape.input_dim as u32,output_dim: shape.output_dim as u32,@@ -79,35 +81,31 @@ fn run_benchmark(let b_byte_count = shape.output_dim * shape.input_dim * elem_size;let d_byte_count = shape.batch * shape.output_dim * elem_size;- let (a_buffer, b_buffer, mut d_buffer) = match (- context.device.new_buffer(a_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),- context.device.new_buffer(b_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),- context.device.new_buffer(d_byte_count, MTLResourceOptions::STORAGE_MODE_SHARED),- ) {- (Some(a), Some(b), Some(d)) => (a, b, d),- _ => return Err(BenchError::BufferAllocation),- };+ let mut a_allocation =+ context.create_allocation(a_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;+ let mut b_allocation =+ context.create_allocation(b_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;+ let mut d_allocation =+ context.create_allocation(d_byte_count, AllocationType::Global).map_err(|_| BenchError::BufferAllocation)?;- fill_buffer_random(&a_buffer, a_byte_count);- fill_buffer_random(&b_buffer, b_byte_count);+ fill_buffer_random(&mut a_allocation, a_byte_count);+ fill_buffer_random(&mut b_allocation, b_byte_count);for iteration in 0..WARMUP_ITERATIONS {- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {- BenchError::Warmup {+ encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation).map_err(+ |source| BenchError::Warmup {iteration,source: Box::new(source),- }- })?;+ },+ )?;}let mut gpu_time_total_ms = 0.0;for iteration in 0..BENCHMARK_ITERATIONS {- let gpu_ms =- encode_and_run(context, &mut kernel, shape, &a_buffer, &b_buffer, &mut d_buffer).map_err(|source| {- BenchError::Benchmark {- iteration,- source: Box::new(source),- }+ let gpu_ms = encode_and_run(context, &mut kernel, shape, &a_allocation, &b_allocation, &mut d_allocation)+ .map_err(|source| BenchError::Benchmark {+ iteration,+ source: Box::new(source),})?;gpu_time_total_ms += gpu_ms;}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rsindex 808c07620..e0f99a8a2 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs@@ -1,9 +1,9 @@#![cfg(metal_backend)]-use std::{cell::RefCell, rc::Rc};+#[macro_use]+#[path = "../../../../common/mod.rs"]+mod common;-use bytemuck;-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use ndarray::{Array, Array3, s};use super::*;@@ -64,56 +64,34 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {apply_swaps_3d(&mut expected_keys, &swaps);apply_swaps_3d(&mut expected_values, &swaps);- let device = &context.device;-- let key_buffer = Rc::new(RefCell::new(- device- .new_buffer_with_data(- bytemuck::cast_slice(key_data.as_slice().unwrap()),- MTLResourceOptions::STORAGE_MODE_SHARED,- )- .expect("Failed to create buffer"),- ));-- let value_buffer = Rc::new(RefCell::new(- device- .new_buffer_with_data(- bytemuck::cast_slice(value_data.as_slice().unwrap()),- MTLResourceOptions::STORAGE_MODE_SHARED,- )- .expect("Failed to create buffer"),- ));-- let kv_layer_data = KVLayerData::<Metal> {- key_buffer: key_buffer.clone(),- key_shape: [seq_len, num_heads, head_dim],- value_buffer: value_buffer.clone(),- value_shape: [seq_len, num_heads, head_dim],- };+ let mut key_allocation = common::helpers::alloc_allocation_with_data(context, key_data.as_slice().unwrap());+ let mut value_allocation = common::helpers::alloc_allocation_with_data(context, value_data.as_slice().unwrap());let mut encoder = Encoder::new(context).unwrap();- match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut encoder) {- Ok(_) => {},- Err(e) => {- println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);- return;- },+ {+ let mut kv_layers = [KVLayerData::<Metal> {+ key_allocation: &mut key_allocation,+ key_shape: [seq_len, num_heads, head_dim],+ value_allocation: &mut value_allocation,+ value_shape: [seq_len, num_heads, head_dim],+ }];+ match kv_cache_update.encode(&mut kv_layers, &source_indices, &destination_indices, &mut encoder) {+ Ok(_) => {},+ Err(e) => {+ println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);+ return;+ },+ }}encoder.end_encoding().submit().wait_until_completed().unwrap();- let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;- let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;-- let total_elems = num_heads * seq_len * head_dim;-- let key_result_slice = unsafe { std::slice::from_raw_parts(key_result_ptr, total_elems) };- let value_result_slice = unsafe { std::slice::from_raw_parts(value_result_ptr, total_elems) };-- let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_result_slice.to_vec())+ let key_values: Vec<f32> = common::helpers::allocation_to_vec(&key_allocation);+ let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_values).expect("Failed to convert key result to ndarray");- let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_result_slice.to_vec())+ let value_values: Vec<f32> = common::helpers::allocation_to_vec(&value_allocation);+ let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_values).expect("Failed to convert value result to ndarray");println!("Original keys tokens 0,14 head 0:");diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsindex 8120fecd1..248dd002a 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs@@ -9,11 +9,11 @@ use half::{bf16, f16};use crate::{DataType,backends::common::{- Backend, Context, DenseBuffer, Encoder,+ Allocation, AllocationType, Backend, Context, Encoder, allocation_as_bytes,gpu_types::QuantizationMode,kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,- QuantizedMatmulType, tests::common::helpers::alloc_buffer_with_data,+ QuantizedMatmulType, tests::common::helpers::alloc_allocation_with_data,},},};@@ -261,23 +261,23 @@ fn generate_test_quant_params(}}-fn buffer_from_f32_slice<B: Backend>(+fn allocation_from_f32_slice<B: Backend>(ctx: &B::Context,dtype: DataType,values: &[f32],-) -> B::DenseBuffer {+) -> Allocation<B> {match dtype {DataType::F16 => {let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();- alloc_buffer_with_data::<B, f16>(ctx, data.as_slice())+ alloc_allocation_with_data::<B, f16>(ctx, data.as_slice())},DataType::BF16 => {let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();- alloc_buffer_with_data::<B, bf16>(ctx, data.as_slice())+ alloc_allocation_with_data::<B, bf16>(ctx, data.as_slice())},- DataType::F32 => alloc_buffer_with_data::<B, f32>(ctx, values),+ DataType::F32 => alloc_allocation_with_data::<B, f32>(ctx, values),other => {- panic!("Unsupported dtype for buffer_from_f32_slice: {:?}", other)+ panic!("Unsupported dtype for allocation_from_f32_slice: {:?}", other)},}}@@ -319,15 +319,17 @@ fn execute_quantized_matmul<B: Backend>(};let x_quant = quantize_slice(&x_f32, data_type);- let w_buf = alloc_buffer_with_data::<B, u8>(ctx, &weights_packed);- let s_buf = buffer_from_f32_slice::<B>(ctx, data_type, ¶ms.scales);+ let w_buf = alloc_allocation_with_data::<B, u8>(ctx, &weights_packed);+ let s_buf = allocation_from_f32_slice::<B>(ctx, data_type, ¶ms.scales);let b_buf = match quantization_type {- QuantizedMatmulType::ZeroPoint => alloc_buffer_with_data::<B, u8>(ctx, ¶ms.zero_points),- QuantizedMatmulType::Mlx => buffer_from_f32_slice::<B>(ctx, data_type, ¶ms.biases),+ QuantizedMatmulType::ZeroPoint => alloc_allocation_with_data::<B, u8>(ctx, ¶ms.zero_points),+ QuantizedMatmulType::Mlx => allocation_from_f32_slice::<B>(ctx, data_type, ¶ms.biases),};- let x_buf = buffer_from_f32_slice::<B>(ctx, data_type, &x_f32);- let mut y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");+ let x_buf = allocation_from_f32_slice::<B>(ctx, data_type, &x_f32);+ let mut y_buf = ctx+ .create_allocation(batch * output_dim * data_type.size_in_bytes(), AllocationType::Global)+ .expect("Failed to create allocation");let kernel = QuantizedMatmulKernelEncodable::<B>::new(&ctx,@@ -350,17 +352,17 @@ fn execute_quantized_matmul<B: Backend>(if iterations > 1 {for _ in 0..3 {let args = QuantizedMatmulArguments {- a_buffer: &x_buf,+ a: &x_buf,a_offset: 0,- b_buffer: &w_buf,- scales_buffer: &s_buf,- zero_points_or_biases_buffer: &b_buf,- output_buffer: &mut y_buf,+ b: &w_buf,+ scales: &s_buf,+ zero_points_or_biases: &b_buf,+ output: &mut y_buf,hadamard_factors: None,batch_dim: batch,};let mut encoder = Encoder::new(ctx).unwrap();- kernel.encode(&mut encoder, args).unwrap();+ kernel.encode(&mut encoder, args);encoder.end_encoding().submit().wait_until_completed().unwrap();}}@@ -368,17 +370,17 @@ fn execute_quantized_matmul<B: Backend>(let start = Instant::now();for _ in 0..iterations {let args = QuantizedMatmulArguments {- a_buffer: &x_buf,+ a: &x_buf,a_offset: 0,- b_buffer: &w_buf,- scales_buffer: &s_buf,- zero_points_or_biases_buffer: &b_buf,- output_buffer: &mut y_buf,+ b: &w_buf,+ scales: &s_buf,+ zero_points_or_biases: &b_buf,+ output: &mut y_buf,hadamard_factors: None,batch_dim: batch,};let mut encoder = Encoder::new(ctx).unwrap();- kernel.encode(&mut encoder, args).unwrap();+ kernel.encode(&mut encoder, args);encoder.end_encoding().submit().wait_until_completed().unwrap();}let elapsed = start.elapsed();@@ -402,20 +404,14 @@ fn execute_quantized_matmul<B: Backend>(let y_out_f32: Vec<f32> = match data_type {DataType::F16 => {- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f16;- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };+ let y_out = bytemuck::cast_slice::<u8, f16>(allocation_as_bytes(&y_buf));y_out.iter().map(|&v| v.to_f32()).collect()},DataType::BF16 => {- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const bf16;- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };+ let y_out = bytemuck::cast_slice::<u8, bf16>(allocation_as_bytes(&y_buf));y_out.iter().map(|&v| v.to_f32()).collect()},- DataType::F32 => {- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f32;- let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };- y_out.to_vec()- },+ DataType::F32 => bytemuck::cast_slice::<u8, f32>(allocation_as_bytes(&y_buf)).to_vec(),other => panic!("Unsupported dtype for validation: {:?}", other),};diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rsindex c261ae43e..85ad712af 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs@@ -1,14 +1,7 @@-use std::{- mem::size_of,- ops::{Deref, DerefMut},-};--use backend_uzu::backends::common::{Buffer, DenseBuffer};-use crate::{- ArrayContextExt, DataType,+ DataType, allocation_to_vec,backends::common::{- Backend, Context, Encoder, Kernels,+ Allocation, Backend, Context, Encoder, Kernels,gpu_types::ActivationType,kernel::{Conv1dScanKernel,@@ -21,30 +14,11 @@ use crate::{#[path = "../../../../common/mod.rs"]mod common;-fn write_buffer<B: Backend>(- buf: &B::DenseBuffer,- data: &[f32],-) {- unsafe {- std::ptr::copy_nonoverlapping(data.as_ptr(), buf.cpu_ptr().as_ptr() as *mut f32, data.len());- }-}--fn read_buffer<B: Backend>(- buf: &B::DenseBuffer,- len: usize,-) -> Vec<f32> {- let mut out = vec![0.0f32; len];- unsafe {- std::ptr::copy_nonoverlapping(buf.cpu_ptr().as_ptr() as *const f32, out.as_mut_ptr(), len);- }- out-}--fn zero_buffer<B: Backend>(buf: &B::DenseBuffer) {- unsafe {- std::ptr::write_bytes(buf.cpu_ptr().as_ptr(), 0, buf.size());- }+fn allocation_from_slice<B: Backend, T: crate::ArrayElement>(+ context: &B::Context,+ data: &[T],+) -> Allocation<B> {+ common::helpers::alloc_allocation_with_data(context, data)}fn ssd_prefill_cpu_reference(@@ -105,7 +79,6 @@ fn ssd_prefill_cpu_reference((y_out, state)}-#[allow(dead_code)]struct SSDPrefillFixture {suffix_len: usize,num_heads: usize,@@ -113,9 +86,6 @@ struct SSDPrefillFixture {state_dim: usize,group_size: i32,total_x: usize,- total_dt: usize,- total_cb: usize,- total_state: usize,x_strides: [usize; 3],dt_strides: [usize; 2],cb_strides: [usize; 3],@@ -163,9 +133,6 @@ impl SSDPrefillFixture {state_dim,group_size,total_x,- total_dt,- total_cb,- total_state,x_strides,dt_strides,cb_strides,@@ -187,56 +154,24 @@ fn run_prefill_kernel_mode<B: Backend>(fixture: &SSDPrefillFixture,mode: SSDPrefillMode,) -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {- let x_array = ctx.create_array_from(&[fixture.x_data.len()], &fixture.x_data, "");- let x_array_buf = x_array.buffer();- let x_array_buf_borrow = x_array_buf.borrow();- let x_buf = x_array_buf_borrow.deref();-- let dt_array = ctx.create_array_from(&[fixture.dt_data.len()], &fixture.dt_data, "");- let dt_array_buf = dt_array.buffer();- let dt_array_buf_borrow = dt_array_buf.borrow();- let dt_buf = dt_array_buf_borrow.deref();-- let b_array = ctx.create_array_from(&[fixture.b_data.len()], &fixture.b_data, "");- let b_array_buf = b_array.buffer();- let b_array_buf_borrow = b_array_buf.borrow();- let b_buf = b_array_buf_borrow.deref();-- let c_array = ctx.create_array_from(&[fixture.c_data.len()], &fixture.c_data, "");- let c_array_buf = c_array.buffer();- let c_array_buf_borrow = c_array_buf.borrow();- let c_buf = c_array_buf_borrow.deref();-- let d_array = ctx.create_array_from(&[fixture.d_data.len()], &fixture.d_data, "");- let d_array_buf = d_array.buffer();- let d_array_buf_borrow = d_array_buf.borrow();- let d_buf = d_array_buf_borrow.deref();-- let z_array = ctx.create_array_from(&[fixture.z_data.len()], &fixture.z_data, "");- let z_array_buf = z_array.buffer();- let z_array_buf_borrow = z_array_buf.borrow();- let z_buf = z_array_buf_borrow.deref();-- let state_array = ctx.create_array_from(&[fixture.state_init.len()], &fixture.state_init, "");- let state_array_buf = state_array.buffer();- let mut state_array_buf_borrow = state_array_buf.borrow_mut();- let state_buf = state_array_buf_borrow.deref_mut();-- let y = vec![0f32; fixture.total_x];- let y_array = ctx.create_array_from(&[y.len()], &y, "");- let y_array_buf = y_array.buffer();- let mut y_array_buf_borrow = y_array_buf.borrow_mut();- let y_buf = y_array_buf_borrow.deref_mut();-- let args = SSDPrefillArguments::<B> {- x: x_buf,- dt: dt_buf,- b: b_buf,- c: c_buf,- d: d_buf,- z: z_buf,- state: state_buf,- y: y_buf,+ let x_buf = allocation_from_slice::<B, _>(ctx, &fixture.x_data);+ let dt_buf = allocation_from_slice::<B, _>(ctx, &fixture.dt_data);+ let b_buf = allocation_from_slice::<B, _>(ctx, &fixture.b_data);+ let c_buf = allocation_from_slice::<B, _>(ctx, &fixture.c_data);+ let d_buf = allocation_from_slice::<B, _>(ctx, &fixture.d_data);+ let z_buf = allocation_from_slice::<B, _>(ctx, &fixture.z_data);+ let mut state_buf = allocation_from_slice::<B, _>(ctx, &fixture.state_init);+ let mut y_buf = allocation_from_slice::<B, _>(ctx, &vec![0f32; fixture.total_x]);++ let args = SSDPrefillArguments {+ x: &x_buf,+ dt: &dt_buf,+ b: &b_buf,+ c: &c_buf,+ d: &d_buf,+ z: &z_buf,+ state: &mut state_buf,+ y: &mut y_buf,suffix_len: fixture.suffix_len,group_size: fixture.group_size as u32,state_size: fixture.state_dim as u32,@@ -252,8 +187,8 @@ fn run_prefill_kernel_mode<B: Backend>(kernel.encode(&mut encoder, args, mode);encoder.end_encoding().submit().wait_until_completed().unwrap();- let y_vec = read_buffer::<B>(y_buf, fixture.total_x);- let state_vec = read_buffer::<B>(state_buf, fixture.total_state);+ let y_vec = allocation_to_vec::<B, f32>(&y_buf);+ let state_vec = allocation_to_vec::<B, f32>(&state_buf);(y_vec, state_vec, None)}@@ -275,106 +210,61 @@ fn run_conv_scan_once<B: Backend>(let _total_w = channels * kernel_size as usize;let total_state = channels * tap_count;- let y_array = if alias_io {- ctx.create_array_from(&[x_data.len()], x_data, "")+ let mut y_buf = if alias_io {+ allocation_from_slice::<B, _>(ctx, x_data)} else {- ctx.create_array_zeros(&[total_x], DataType::F32, "")+ allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_x])};- let b_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");- let c_out_array = ctx.create_array_zeros(&[total_x], DataType::F32, "");-- let w_array = ctx.create_array_from(&[w_data.len()], w_data, "");- let b_array = ctx.create_array_from(&[b_data.len()], b_data, "");-- let state_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");- let scratch_array = ctx.create_array_uninitialized(&[total_state], DataType::F32, "");-- {- let state_buf = state_array.buffer();- let state_borrow = state_buf.borrow();- write_buffer::<B>(state_borrow.deref(), state_init);- if use_scratch && tap_count > 0 {- let scratch_buf = scratch_array.buffer();- let scratch_borrow = scratch_buf.borrow();- zero_buffer::<B>(scratch_borrow.deref());- }- }+ let mut b_out_buf = allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_x]);+ let mut c_out_buf = allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_x]);+ let w_buf = allocation_from_slice::<B, _>(ctx, w_data);+ let b_buf = allocation_from_slice::<B, _>(ctx, b_data);+ let mut state_buf = allocation_from_slice::<B, _>(ctx, state_init);+ let scratch_buf = allocation_from_slice::<B, _>(ctx, &vec![0.0f32; total_state]);let padded_len = tap_count + suffix_len;- let padded_array = ctx.create_array_uninitialized(&[padded_len * channels], DataType::F32, "");- {- let padded_buf = padded_array.buffer();- let padded_borrow = padded_buf.borrow();- let mut host = vec![0.0f32; padded_len * channels];- for tap in 0..tap_count {- for ch in 0..channels {- host[tap * channels + ch] = state_init[ch * tap_count + tap];- }+ let mut padded_host = vec![0.0f32; padded_len * channels];+ for tap in 0..tap_count {+ for ch in 0..channels {+ padded_host[tap * channels + ch] = state_init[ch * tap_count + tap];}- for token in 0..suffix_len {- for ch in 0..channels {- host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];- }+ }+ for token in 0..suffix_len {+ for ch in 0..channels {+ padded_host[(tap_count + token) * channels + ch] = x_data[token * channels + ch];}- write_buffer::<B>(padded_borrow.deref(), &host);}+ let padded_buf = allocation_from_slice::<B, _>(ctx, &padded_host);let mut encoder = Encoder::new(ctx).unwrap();- {- let padded_buf = padded_array.buffer();- let padded_borrow = padded_buf.borrow();- let w_buf = w_array.buffer();- let w_borrow = w_buf.borrow();- let b_buf = b_array.buffer();- let b_borrow = b_buf.borrow();- let y_buf = y_array.buffer();- let mut y_borrow = y_buf.borrow_mut();- let b_out_buf = b_out_array.buffer();- let mut b_out_borrow = b_out_buf.borrow_mut();- let c_out_buf = c_out_array.buffer();- let mut c_out_borrow = c_out_buf.borrow_mut();- let state_buf = state_array.buffer();- let mut state_borrow = state_buf.borrow_mut();-- kernel.encode(- padded_borrow.deref(),- w_borrow.deref(),- Some(b_borrow.deref()),- y_borrow.deref_mut(),- b_out_borrow.deref_mut(),- c_out_borrow.deref_mut(),- state_borrow.deref_mut(),- suffix_len as u32,- kernel_size as u32,- channels as u32,- tap_count as u32,- channels as u32,- channels as u32,- 0u32,- ActivationType::SILU,- &mut encoder,- );- }+ kernel.encode(+ &padded_buf,+ &w_buf,+ Some(&b_buf),+ &mut y_buf,+ &mut b_out_buf,+ &mut c_out_buf,+ &mut state_buf,+ suffix_len as u32,+ kernel_size as u32,+ channels as u32,+ tap_count as u32,+ channels as u32,+ channels as u32,+ 0u32,+ ActivationType::SILU,+ &mut encoder,+ );if use_scratch && tap_count > 0 {let bytes = channels * tap_count * size_of::<f32>();- if bytes > 0 {- let scratch_buf = scratch_array.buffer();- let scratch_borrow = scratch_buf.borrow();- let state_buf = state_array.buffer();- let mut state_borrow = state_buf.borrow_mut();- encoder.encode_copy(scratch_borrow.deref(), 0..bytes, state_borrow.deref_mut(), 0..bytes);- }+ encoder.encode_copy(&scratch_buf, 0..bytes, &mut state_buf, 0..bytes);}encoder.end_encoding().submit().wait_until_completed().unwrap();- let y_buf = y_array.buffer();- let y_borrow = y_buf.borrow();- let state_buf = state_array.buffer();- let state_borrow = state_buf.borrow();- let y_vec = read_buffer::<B>(y_borrow.deref(), total_x);- let state_vec = read_buffer::<B>(state_borrow.deref(), total_state);+ let y_vec = allocation_to_vec::<B, f32>(&y_buf);+ let state_vec = allocation_to_vec::<B, f32>(&state_buf);(y_vec, state_vec)}diff --git a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rsindex b1c7a5e47..0122377f3 100644--- a/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/attention_test.rs@@ -1,18 +1,13 @@#![cfg(metal_backend)]-use std::mem::size_of;--use bytemuck;-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use ndarray::{Array4, s};-use objc2::{rc::Retained, runtime::ProtocolObject};use test_tag::tag;use crate::{DataType,backends::{common::{- Backend, Context, Encoder, Kernels,+ Allocation, Backend, Context, Encoder, Kernels,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,attention::{AttentionGemmArguments, AttentionGemmBlock},@@ -22,227 +17,199 @@ use crate::{},};+#[path = "../../common/mod.rs"]+mod common;++use common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, submit_encoder};+fn reference_attention(- queries: &Array4<f32>, // [batch, num_heads, seq_len, head_dim]- keys: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]- values: &Array4<f32>, // [batch, num_kv_heads, seq_len, head_dim]+ queries: &Array4<f32>,+ keys: &Array4<f32>,+ values: &Array4<f32>,sinks: Option<&[f32]>,scale: f32,is_causal: bool,) -> Array4<f32> {- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();- let (_, num_kv_heads, kv_seq_len, _) = keys.dim();- let n_repeats = num_heads / num_kv_heads;-- let scaled_queries = queries.mapv(|x| x * scale);-- let mut output = Array4::zeros((batch_size, num_heads, seq_len, head_dim));-- for b in 0..batch_size {- if n_repeats > 1 {- for kv_head in 0..num_kv_heads {- for repeat in 0..n_repeats {- let q_head = kv_head * n_repeats + repeat;-- let q = scaled_queries.slice(s![b, q_head, .., ..]).to_owned(); // [L, D]- let k = keys.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]- let v = values.slice(s![b, kv_head, .., ..]).to_owned(); // [L_kv, D]-- // Efficient matrix multiplication: scores = Q @ K^T- let mut scores = q.dot(&k.t());-- if is_causal {- for i in 0..seq_len {- for j in (i + 1)..kv_seq_len {- scores[[i, j]] = f32::NEG_INFINITY;- }- }- }-- // Softmax with optional sink logit- for i in 0..seq_len {- let sink_logit = sinks.map(|s| s[q_head]);- let row = scores.row(i);- let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));-- let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);- for j in 0..kv_seq_len {- scores[[i, j]] = (scores[[i, j]] - max_score).exp();- sum_exp += scores[[i, j]];- }- scores.row_mut(i).mapv_inplace(|x| x / sum_exp);- }-- // Efficient matrix multiplication: output = scores @ V- let head_output = scores.dot(&v);- output.slice_mut(s![b, q_head, .., ..]).assign(&head_output);+ let (batch_size, num_heads, sequence_length, head_dim) = queries.dim();+ let (_, num_kv_heads, kv_sequence_length, _) = keys.dim();+ let repeats_per_kv_head = num_heads / num_kv_heads;+ let scaled_queries = queries.mapv(|value| value * scale);+ let mut output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));++ for batch_index in 0..batch_size {+ if repeats_per_kv_head > 1 {+ for kv_head_index in 0..num_kv_heads {+ for repeat_index in 0..repeats_per_kv_head {+ let head_index = kv_head_index * repeats_per_kv_head + repeat_index;+ assign_attention_head_output(+ &mut output,+ &scaled_queries,+ keys,+ values,+ sinks,+ is_causal,+ batch_index,+ head_index,+ kv_head_index,+ sequence_length,+ kv_sequence_length,+ );}}} else {- for h in 0..num_heads {- let q = scaled_queries.slice(s![b, h, .., ..]).to_owned(); // [L, D]- let k = keys.slice(s![b, h, .., ..]).to_owned(); // [L_kv, D]- let v = values.slice(s![b, h, .., ..]).to_owned(); // [L_kv, D]-- // Efficient matrix multiplication: scores = Q @ K^T- let mut scores = q.dot(&k.t());-- if is_causal {- for i in 0..seq_len {- for j in (i + 1)..kv_seq_len {- scores[[i, j]] = f32::NEG_INFINITY;- }- }- }+ for head_index in 0..num_heads {+ assign_attention_head_output(+ &mut output,+ &scaled_queries,+ keys,+ values,+ sinks,+ is_causal,+ batch_index,+ head_index,+ head_index,+ sequence_length,+ kv_sequence_length,+ );+ }+ }+ }- // Softmax with optional sink logit- for i in 0..seq_len {- let sink_logit = sinks.map(|s| s[h]);- let row = scores.row(i);- let max_score = row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |acc, &x| acc.max(x));-- let mut sum_exp = sink_logit.map(|sink| (sink - max_score).exp()).unwrap_or(0.0);- for j in 0..kv_seq_len {- scores[[i, j]] = (scores[[i, j]] - max_score).exp();- sum_exp += scores[[i, j]];- }- scores.row_mut(i).mapv_inplace(|x| x / sum_exp);- }+ output+}- // Efficient matrix multiplication: output = scores @ V- let head_output = scores.dot(&v);- output.slice_mut(s![b, h, .., ..]).assign(&head_output);+fn assign_attention_head_output(+ output: &mut Array4<f32>,+ scaled_queries: &Array4<f32>,+ keys: &Array4<f32>,+ values: &Array4<f32>,+ sinks: Option<&[f32]>,+ is_causal: bool,+ batch_index: usize,+ head_index: usize,+ kv_head_index: usize,+ sequence_length: usize,+ kv_sequence_length: usize,+) {+ let queries = scaled_queries.slice(s![batch_index, head_index, .., ..]).to_owned();+ let keys = keys.slice(s![batch_index, kv_head_index, .., ..]).to_owned();+ let values = values.slice(s![batch_index, kv_head_index, .., ..]).to_owned();+ let mut scores = queries.dot(&keys.t());++ if is_causal {+ for query_index in 0..sequence_length {+ for key_index in (query_index + 1)..kv_sequence_length {+ scores[[query_index, key_index]] = f32::NEG_INFINITY;}}}- output+ for query_index in 0..sequence_length {+ let sink_logit = sinks.map(|sink_values| sink_values[head_index]);+ let row = scores.row(query_index);+ let max_score =+ row.iter().fold(sink_logit.unwrap_or(f32::NEG_INFINITY), |current_max, &score| current_max.max(score));++ let mut sum_exp = sink_logit.map(|sink_value| (sink_value - max_score).exp()).unwrap_or(0.0);+ for key_index in 0..kv_sequence_length {+ scores[[query_index, key_index]] = (scores[[query_index, key_index]] - max_score).exp();+ sum_exp += scores[[query_index, key_index]];+ }+ scores.row_mut(query_index).mapv_inplace(|value| value / sum_exp);+ }++ let head_output = scores.dot(&values);+ output.slice_mut(s![batch_index, head_index, .., ..]).assign(&head_output);}fn create_test_data(batch_size: usize,num_heads: usize,num_kv_heads: usize,- seq_len: usize,+ sequence_length: usize,head_dim: usize,seed: u64,) -> (Array4<f32>, Array4<f32>, Array4<f32>) {use rand::{RngExt, SeedableRng, rngs::StdRng};- let mut rng = StdRng::seed_from_u64(seed);-- let queries = Array4::from_shape_fn((batch_size, num_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));-- let keys = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));+ let mut random = StdRng::seed_from_u64(seed);- let values = Array4::from_shape_fn((batch_size, num_kv_heads, seq_len, head_dim), |_| rng.random_range(-0.5..0.5));+ let queries =+ Array4::from_shape_fn((batch_size, num_heads, sequence_length, head_dim), |_| random.random_range(-0.5..0.5));+ let keys = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {+ random.random_range(-0.5..0.5)+ });+ let values = Array4::from_shape_fn((batch_size, num_kv_heads, sequence_length, head_dim), |_| {+ random.random_range(-0.5..0.5)+ });(queries, keys, values)}-/// Convert ndarray to Metal buffer layout expected by our kernel-fn create_query_buffer(+fn create_query_allocation(queries: &Array4<f32>,context: &<Metal as Backend>::Context,-) -> Retained<ProtocolObject<dyn MTLBuffer>> {- let (_batch_size, num_heads, seq_len, head_dim) = queries.dim();-- // Our kernel expects queries layout: [num_heads, seq_len, head_dim]- let mut query_data = vec![0.0f32; num_heads * seq_len * head_dim];-- for h in 0..num_heads {- for t in 0..seq_len {- for d in 0..head_dim {- let idx = h * seq_len * head_dim + t * head_dim + d;- query_data[idx] = queries[[0, h, t, d]];+) -> Allocation<Metal> {+ let (_batch_size, num_heads, sequence_length, head_dim) = queries.dim();+ let mut values = vec![0.0_f32; num_heads * sequence_length * head_dim];++ for head_index in 0..num_heads {+ for sequence_index in 0..sequence_length {+ for dim_index in 0..head_dim {+ let flat_index = head_index * sequence_length * head_dim + sequence_index * head_dim + dim_index;+ values[flat_index] = queries[[0, head_index, sequence_index, dim_index]];}}}- context- .device- .new_buffer_with_data(bytemuck::cast_slice(&query_data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")+ alloc_allocation_with_data(context, &values)}-fn create_key_cache_buffer(- keys: &Array4<f32>,- max_seq_len: usize,- context: &<Metal as Backend>::Context,-) -> Retained<ProtocolObject<dyn MTLBuffer>> {- let (_batch_size, num_kv_heads, seq_len, head_dim) = keys.dim();-- // Our kernel expects key cache layout: [max_seq_len, num_kv_heads, head_dim]- let mut key_cache_data = vec![0.0f32; max_seq_len * num_kv_heads * head_dim];-- for t in 0..seq_len {- for h in 0..num_kv_heads {- for d in 0..head_dim {- let idx = t * num_kv_heads * head_dim + h * head_dim + d;- key_cache_data[idx] = keys[[0, h, t, d]];- }- }- }-- context- .device- .new_buffer_with_data(bytemuck::cast_slice(&key_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")-}--fn create_value_cache_buffer(+fn create_attention_cache_allocation(values: &Array4<f32>,- max_seq_len: usize,+ max_sequence_length: usize,context: &<Metal as Backend>::Context,-) -> Retained<ProtocolObject<dyn MTLBuffer>> {- let (_batch_size, num_kv_heads, seq_len, head_dim) = values.dim();-- // Our kernel expects value cache layout: [max_seq_len, num_kv_heads, head_dim]- let mut value_cache_data = vec![0.0f32; max_seq_len * num_kv_heads * head_dim];-- for t in 0..seq_len {- for h in 0..num_kv_heads {- for d in 0..head_dim {- let idx = t * num_kv_heads * head_dim + h * head_dim + d;- value_cache_data[idx] = values[[0, h, t, d]];+) -> Allocation<Metal> {+ let (_batch_size, num_kv_heads, sequence_length, head_dim) = values.dim();+ let mut cache = vec![0.0_f32; max_sequence_length * num_kv_heads * head_dim];++ for sequence_index in 0..sequence_length {+ for head_index in 0..num_kv_heads {+ for dim_index in 0..head_dim {+ let flat_index = sequence_index * num_kv_heads * head_dim + head_index * head_dim + dim_index;+ cache[flat_index] = values[[0, head_index, sequence_index, dim_index]];}}}- context- .device- .new_buffer_with_data(bytemuck::cast_slice(&value_cache_data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")+ alloc_allocation_with_data(context, &cache)}-fn create_sinks_buffer(+fn create_sinks_allocation(sinks: &[f32],context: &<Metal as Backend>::Context,-) -> Retained<ProtocolObject<dyn MTLBuffer>> {- context- .device- .new_buffer_with_data(bytemuck::cast_slice(&sinks), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")+) -> Allocation<Metal> {+ alloc_allocation_with_data(context, sinks)}fn convert_kernel_output(- output_slice: &[f32],+ output: &[f32],batch_size: usize,num_heads: usize,- seq_len: usize,+ sequence_length: usize,head_dim: usize,) -> Array4<f32> {- let mut kernel_output = Array4::zeros((batch_size, num_heads, seq_len, head_dim));- for h in 0..num_heads {- for t in 0..seq_len {- for d in 0..head_dim {- // Metal kernel output layout: o_offset = q_seq_idx * num_heads + head_idx- let o_offset = t * num_heads + h;- let idx = o_offset * head_dim + d;- kernel_output[[0, h, t, d]] = output_slice[idx];+ let mut kernel_output = Array4::zeros((batch_size, num_heads, sequence_length, head_dim));++ for head_index in 0..num_heads {+ for sequence_index in 0..sequence_length {+ for dim_index in 0..head_dim {+ let flat_index = (sequence_index * num_heads + head_index) * head_dim + dim_index;+ kernel_output[[0, head_index, sequence_index, dim_index]] = output[flat_index];}}}+kernel_output}@@ -258,16 +225,11 @@ fn run_single_pass_attention(let (batch_size, num_heads, seq_len, head_dim) = queries.dim();let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();- let query_buffer = create_query_buffer(queries, context);- let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);- let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);-- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));-- let mut output_buffer = context- .device- .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let query_buffer = create_query_allocation(queries, context);+ let key_cache_buffer = create_attention_cache_allocation(keys, seq_len, context);+ let value_cache_buffer = create_attention_cache_allocation(values, seq_len, context);+ let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);let mut encoder = Encoder::new(context).expect("Failed to create encoder");@@ -284,77 +246,63 @@ fn run_single_pass_attention((num_kv_heads * head_dim) as u32,None,scale,- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,+ None::<&Allocation<Metal>>,None,sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,&mut encoder,);- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };+ submit_encoder(encoder);- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);+ let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);+ let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);Ok(kernel_output)}-fn run_single_pass_attention_with_is_causal(- kernel: &<<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel,+fn create_single_pass_kernel(context: &<Metal as Backend>::Context,- queries: &Array4<f32>,- keys: &Array4<f32>,- values: &Array4<f32>,- sinks: Option<&[f32]>,- scale: f32,- _is_causal: bool,-) -> Result<Array4<f32>, Box<dyn std::error::Error>> {- let (batch_size, num_heads, seq_len, head_dim) = queries.dim();- let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();-- let query_buffer = create_query_buffer(queries, context);- let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);- let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);-- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));-- let mut output_buffer = context- .device- .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut encoder = Encoder::new(context).expect("Failed to create encoder");-- kernel.encode(- &query_buffer,- &key_cache_buffer,- &value_cache_buffer,- &mut output_buffer,- (num_heads / num_kv_heads) as u32,- seq_len as u32,- head_dim as u32,- (num_kv_heads * head_dim) as u32,+ head_dim: usize,+ has_sinks: bool,+ is_causal: bool,+) -> <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel {+ <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(+ context,+ DataType::F32,head_dim as u32,- (num_kv_heads * head_dim) as u32,- None,- scale,- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,- None,- sinks_buffer.as_ref().map(|b| b),- num_heads as u32,- seq_len as u32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };-- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);+ has_sinks,+ false,+ is_causal,+ false,+ false,+ )+ .expect("Failed to create attention single pass kernel")+}- Ok(kernel_output)+fn create_two_pass_kernels(+ context: &<Metal as Backend>::Context,+ head_dim: usize,+ is_causal: bool,+) -> (+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel,+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel,+) {+ (+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(+ context,+ DataType::F32,+ head_dim as u32,+ false,+ false,+ is_causal,+ false,+ false,+ )+ .expect("Failed to create AttentionTwoPass1Kernel"),+ <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, DataType::F32, head_dim as u32)+ .expect("Failed to create AttentionTwoPass2Kernel"),+ )}fn run_gemm_attention(@@ -370,26 +318,23 @@ fn run_gemm_attention(let (batch_size, num_heads, seq_len, head_dim) = queries.dim();let (_batch_size, num_kv_heads, _seq_len, _head_dim) = keys.dim();- let query_buffer = create_query_buffer(queries, context);- let key_cache_buffer = create_key_cache_buffer(keys, seq_len, context);- let value_cache_buffer = create_value_cache_buffer(values, seq_len, context);+ let query_allocation = create_query_allocation(queries, context);+ let key_allocation = create_attention_cache_allocation(keys, seq_len, context);+ let value_allocation = create_attention_cache_allocation(values, seq_len, context);- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));+ let sinks_allocation = sinks.map(|sinks| create_sinks_allocation(sinks, context));- let mut output_buffer = context- .device- .new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut output_allocation = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);let mut encoder = Encoder::new(context).expect("Failed to create encoder");let args = AttentionGemmArguments {- queries_buffer: &query_buffer,- keys_buffer: &key_cache_buffer,- values_buffer: &value_cache_buffer,- output_buffer: &mut output_buffer,- trie_buffer: None,- sinks_buffer: sinks_buffer.as_ref(),+ queries: &query_allocation,+ keys: &key_allocation,+ values: &value_allocation,+ output: &mut output_allocation,+ trie: None,+ sinks: sinks_allocation.as_ref(),num_heads,num_groups: num_kv_heads,suffix_length: seq_len,@@ -407,13 +352,12 @@ fn run_gemm_attention(v_seq_stride: (num_kv_heads * head_dim) as u64,};- kernel.encode(context, &mut encoder, args)?;- encoder.end_encoding().submit().wait_until_completed().unwrap();+ kernel.encode(&mut encoder, args)?;+ submit_encoder(encoder);- let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };+ let output: Vec<f32> = allocation_to_vec(&output_allocation);- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);+ let kernel_output = convert_kernel_output(&output, batch_size, num_heads, seq_len, head_dim);Ok(kernel_output)}@@ -424,36 +368,8 @@ fn compare_results(tolerance: f32,test_name: &str,) -> Result<(), String> {- let max_diff = kernel_output.iter().zip(reference_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);-- let relative_error = kernel_output- .iter()- .zip(reference_output.iter())- .map(|(a, b)| {- if b.abs() > 1e-6 {- (a - b).abs() / b.abs()- } else {- (a - b).abs()- }- })- .fold(0.0f32, f32::max);-- println!("Max absolute difference ({}): {}", test_name, max_diff);- println!("Max relative error ({}): {}", test_name, relative_error);-- println!("Debug - Sample values ({}):", test_name);- println!("Kernel output [0,0,0,0:4]: {:?}", &kernel_output.slice(s![0, 0, 0, 0..4]).to_vec());- println!("Reference output [0,0,0,0:4]: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());- println!("Kernel output [0,0,1,0:4]: {:?}", &kernel_output.slice(s![0, 0, 1, 0..4]).to_vec());- println!("Reference output [0,0,1,0:4]: {:?}", &reference_output.slice(s![0, 0, 1, 0..4]).to_vec());-- let kernel_max = kernel_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));- let kernel_min = kernel_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));- let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));- let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));-- println!("Kernel output range: [{}, {}]", kernel_min, kernel_max);- println!("Reference output range: [{}, {}]", ref_min, ref_max);+ let max_diff =+ kernel_output.iter().zip(reference_output.iter()).map(|(lhs, rhs)| (lhs - rhs).abs()).fold(0.0_f32, f32::max);if max_diff >= tolerance {return Err(format!(@@ -462,7 +378,6 @@ fn compare_results());}- println!("✓ {} test passed!", test_name);Ok(())}@@ -478,39 +393,12 @@ fn test_single_pass_attention_basic() {let scale = 1.0 / (head_dim as f32).sqrt();let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);-- println!("Testing reference implementation...");let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);+ let kernel = create_single_pass_kernel(&context, head_dim, false, false);+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)+ .expect("single-pass attention");- let ref_max = reference_output.iter().fold(0.0f32, |a, &b| a.max(b.abs()));- let ref_min = reference_output.iter().fold(f32::INFINITY, |a, &b| a.min(b.abs()));- println!("Reference output range: [{}, {}]", ref_min, ref_max);- println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());-- let is_causal = false; // Non-causal attention for this test- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &context,- DataType::F32,- head_dim as u32,- false,- false,- is_causal,- false,- false,- )- .expect("Failed to create attention single pass metal");-- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run single-pass attention: {:?}", e);- },- };-- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention") {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention").unwrap();}#[test]@@ -527,17 +415,11 @@ fn test_gemm_attention_basic() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);let kernel = AttentionGemmBlock::new(DataType::F32);-let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);+ let kernel_output = run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, false)+ .expect("run gemm attention");- let kernel_output =- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false)- .expect("run gemm attention");-- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention") {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention").unwrap();}#[test]@@ -554,17 +436,11 @@ fn test_gemm_attention_f32_head_dim_128() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);let kernel = AttentionGemmBlock::new(DataType::F32);-let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);+ let kernel_output = run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, false)+ .expect("run gemm attention f32 head_dim=128");- let kernel_output =- run_gemm_attention(&kernel, &context, &queries, &keys, &values, None, scale, /*is_causal=*/ false)- .expect("run gemm attention f32 head_dim=128");-- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Gemm attention f32 head_dim=128") {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 1e-2, "Gemm attention f32 head_dim=128").unwrap();}#[test]@@ -581,31 +457,13 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);- // CPU referencelet reference_output = reference_attention(&queries, &keys, &values, None, scale, is_causal);-- // vector attention path (single-pass)- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &context,- DataType::F32,- head_dim as u32,- false,- false,- is_causal,- false,- false,- )- .expect("Failed to create attention single pass metal");- let vector_output =- run_single_pass_attention_with_is_causal(&kernel, &context, &queries, &keys, &values, None, scale, is_causal)- .expect("run vector attention");-- // matrix attention path (gemm)+ let kernel = create_single_pass_kernel(&context, head_dim, false, is_causal);+ let vector_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)+ .expect("run vector attention");let gemm_kernel = AttentionGemmBlock::new(DataType::F32);let matrix_output = run_gemm_attention(&gemm_kernel, &context, &queries, &keys, &values, None, scale, is_causal).expect("run matrix attention");-- // Compare both to CPUlet tol_cpu = 5e-2;compare_results(&vector_output, &reference_output, tol_cpu, "vector single-pass attention vs CPU").unwrap();compare_results(&matrix_output, &reference_output, tol_cpu, "matrix attention vs CPU").unwrap();@@ -613,7 +471,6 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {let max_diff_vector_matrix =vector_output.iter().zip(matrix_output.iter()).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);let tol_vector_matrix = 5e-2;- println!("Max absolute difference (vector single-pass attention vs matrix attention): {}", max_diff_vector_matrix);assert!(max_diff_vector_matrix <= tol_vector_matrix,"vector single-pass attention and matrix attention differ too much: max_diff = {} (tol={})",@@ -636,34 +493,13 @@ fn test_single_pass_attention_with_sinks() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 444);let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();- println!("Using sinks: {:?}", sinks);-- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &context,- DataType::F32,- head_dim as u32,- true,- false,- false,- false,- false,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");+ let kernel = create_single_pass_kernel(&context, head_dim, true, false);let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)+ .expect("single-pass attention with sinks");- let kernel_output =- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run single-pass attention with sinks: {:?}", e);- },- };-- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention with sinks") {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention with sinks").unwrap();}#[test]@@ -680,39 +516,14 @@ fn test_single_pass_attention_with_sinks_long_sequence() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 777);let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();- println!("Long sequence sinks: {:?}", sinks);-- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &context,- DataType::F32,- head_dim as u32,- true,- false,- false,- false,- false,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");+ let kernel = create_single_pass_kernel(&context, head_dim, true, false);let reference_output = reference_attention(&queries, &keys, &values, Some(&sinks), scale, false);+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale)+ .expect("single-pass attention with sinks (long sequence)");- let kernel_output =- match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, Some(&sinks), scale) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run single-pass attention long sequence with sinks: {:?}", e);- },- };-- let tolerance = 5e-2;- if let Err(e) = compare_results(- &kernel_output,- &reference_output,- tolerance,- "Single-pass attention with sinks (long sequence)",- ) {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 5e-2, "Single-pass attention with sinks (long sequence)")+ .unwrap();}#[test]@@ -728,32 +539,12 @@ fn test_single_pass_attention_gqa() {let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);- let is_causal = false; // Non-causal attention for this test- let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel::new(- &context,- DataType::F32,- head_dim as u32,- false,- false,- is_causal,- false,- false,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionSinglePassKernel");-- let kernel_output = match run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run single-pass attention GQA: {:?}", e);- },- };+ let kernel = create_single_pass_kernel(&context, head_dim, false, false);+ let kernel_output = run_single_pass_attention(&kernel, &context, &queries, &keys, &values, None, scale)+ .expect("single-pass attention GQA");let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);-- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Single-pass attention GQA") {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 1e-2, "Single-pass attention GQA").unwrap();}fn run_two_pass_attention(@@ -769,37 +560,19 @@ fn run_two_pass_attention(let (batch_size, num_heads, seq_len, head_dim) = queries.dim();let (_, num_kv_heads, _, _) = keys.dim();- let queries_buffer = create_query_buffer(queries, context);- let keys_buffer = create_key_cache_buffer(keys, seq_len, context);- let values_buffer = create_value_cache_buffer(values, seq_len, context);- let sinks_buffer = sinks.map(|s| create_sinks_buffer(s, context));+ let queries_buffer = create_query_allocation(queries, context);+ let keys_buffer = create_attention_cache_allocation(keys, seq_len, context);+ let values_buffer = create_attention_cache_allocation(values, seq_len, context);+ let sinks_buffer = sinks.map(|sinks| create_sinks_allocation(sinks, context));let total_blocks_count = 32;let partials_size = num_heads * seq_len * total_blocks_count * head_dim;let sums_maxs_size = num_heads * seq_len * total_blocks_count;- let mut partials_buffer = context- .device- .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut sums_buffer = context- .device- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut maxs_buffer = context- .device- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut output_buffer = context- .device- .new_buffer(- num_heads * seq_len * head_dim * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )- .expect("Failed to create buffer");+ let mut partials_buffer = alloc_allocation::<Metal, f32>(context, partials_size);+ let mut sums_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);+ let mut maxs_buffer = alloc_allocation::<Metal, f32>(context, sums_maxs_size);+ let mut output_buffer = alloc_allocation::<Metal, f32>(context, num_heads * seq_len * head_dim);let mut encoder = Encoder::new(context).expect("Failed to create encoder");@@ -820,7 +593,7 @@ fn run_two_pass_attention(scale,num_heads as u32,seq_len as u32,- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,+ None::<&Allocation<Metal>>,None,sinks_buffer.as_ref().map(|b| b),&mut encoder,@@ -834,12 +607,10 @@ fn run_two_pass_attention(seq_len as u32,&mut encoder,);- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };+ submit_encoder(encoder);- let kernel_output = convert_kernel_output(output_slice, batch_size, num_heads, seq_len, head_dim);+ let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);+ let kernel_output = convert_kernel_output(&output_slice, batch_size, num_heads, seq_len, head_dim);Ok(kernel_output)}@@ -860,35 +631,12 @@ fn test_two_pass_attention() {let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(- &context,- DataType::F32,- head_dim as u32,- false,- false,- is_causal,- false,- false,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");- let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(- &context,- DataType::F32,- head_dim as u32,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");+ let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);let kernel_output =- match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run two-pass attention: {:?}", e);- },- };+ run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)+ .expect("two-pass attention");- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention") {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention").unwrap();}#[test]@@ -907,35 +655,12 @@ fn test_two_pass_attention_gqa() {let reference_output = reference_attention(&queries, &keys, &values, None, scale, false);- let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(- &context,- DataType::F32,- head_dim as u32,- false,- false,- is_causal,- false,- false,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel");- let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(- &context,- DataType::F32,- head_dim as u32,- )- .expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel");+ let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);let kernel_output =- match run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale) {- Ok(output) => output,- Err(e) => {- panic!("Failed to run two-pass attention GQA: {:?}", e);- },- };+ run_two_pass_attention(&kernel_pass1, &kernel_pass2, &context, &queries, &keys, &values, None, scale)+ .expect("two-pass attention GQA");- let tolerance = 1e-2;- if let Err(e) = compare_results(&kernel_output, &reference_output, tolerance, "Two-pass attention GQA") {- panic!("{}", e);- }+ compare_results(&kernel_output, &reference_output, 1e-2, "Two-pass attention GQA").unwrap();}#[tag(heavy)]@@ -955,66 +680,27 @@ fn perf_two_pass_attention() {let scale = 1.0 / (head_dim as f32).sqrt();let is_causal = false;- println!(- "Creating test data for two-pass performance test (prefix={}, suffix={})...",- seq_len - suffix_length,- suffix_length- );let (queries, keys, values) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);+ let (kernel_pass1, kernel_pass2) = create_two_pass_kernels(&context, head_dim, is_causal);- let kernel_pass1 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass1Kernel::new(- &context,- DataType::F32,- head_dim as u32,- false,- false,- is_causal,- false,- false,- )- .expect("Failed to create AttentionTwoPass1Kernel");- let kernel_pass2 = <<Metal as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(- &context,- DataType::F32,- head_dim as u32,- )- .expect("Failed to create AttentionTwoPass2Kernel");-- // ---- Create buffers ----- // For realistic inference, we only process queries for the suffix (new tokens)- let queries_suffix = queries.slice(s![.., .., (seq_len - suffix_length).., ..]).to_owned();- let queries_buffer = create_query_buffer(&queries_suffix, &context);- let keys_buffer = create_key_cache_buffer(&keys, seq_len, &context);- let values_buffer = create_value_cache_buffer(&values, seq_len, &context);+ let suffix_start = (seq_len - suffix_length) as usize;+ let queries_suffix = queries.slice(s![.., .., suffix_start.., ..]).to_owned();+ let queries_buffer = create_query_allocation(&queries_suffix, &context);+ let keys_buffer = create_attention_cache_allocation(&keys, seq_len, &context);+ let values_buffer = create_attention_cache_allocation(&values, seq_len, &context);let total_blocks_count = 32;let partials_size = num_heads * suffix_length * total_blocks_count * head_dim;let sums_maxs_size = num_heads * suffix_length * total_blocks_count;- let mut partials_buffer = context- .device- .new_buffer(partials_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let mut sums_buffer = context- .device- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let mut maxs_buffer = context- .device- .new_buffer(sums_maxs_size * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let mut output_buffer = context- .device- .new_buffer(- num_heads * suffix_length * head_dim * std::mem::size_of::<f32>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )- .expect("Failed to create buffer");+ let mut partials_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), partials_size);+ let mut sums_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);+ let mut maxs_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), sums_maxs_size);+ let mut output_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), num_heads * suffix_length * head_dim);- // ---- Launch and time ----let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;+ let sinks_allocation: Option<Allocation<Metal>> = None;kernel_pass1.encode(&queries_buffer,&keys_buffer,@@ -1032,9 +718,9 @@ fn perf_two_pass_attention() {scale,num_heads as u32,suffix_length as u32,- None::<&Retained<ProtocolObject<dyn MTLBuffer>>>,+ None::<&Allocation<Metal>>,None,- sinks_buffer.as_ref().map(|b| b),+ sinks_allocation.as_ref().map(|b| b),&mut encoder,);kernel_pass2.encode(@@ -1046,7 +732,6 @@ fn perf_two_pass_attention() {suffix_length as u32,&mut encoder,);- // Time both host-side and GPU executionlet host_timer = Instant::now();let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;@@ -1062,14 +747,8 @@ fn perf_two_pass_attention() {host_elapsed_ms);- // ---- Sanity check ----- let output_ptr = output_buffer.contents().as_ptr() as *const f32;- let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * suffix_length * head_dim) };-- // Check for NaN/Inf+ let output_slice: Vec<f32> = allocation_to_vec(&output_buffer);for &val in output_slice.iter().take(100) {assert!(val.is_finite(), "Output contains non-finite values");}-- println!("✓ Two-pass attention performance test completed");}diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rsindex 1f2746951..a5e4c8073 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_block_e2e_test.rs@@ -1,7 +1,7 @@use backend_uzu::{DataType,backends::common::{- Backend, DenseBuffer, Encoder, Kernels,+ Backend, Encoder, Kernels,gpu_types::{ActivationType, activation_silu_alpha},kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,@@ -13,7 +13,9 @@ use backend_uzu::{use half::bf16;use rand::{RngExt, SeedableRng, rngs::StdRng};-use crate::encodable_block::mlp::moe::tests::common::helpers::{alloc_buffer, alloc_buffer_with_data, create_context};+use crate::encodable_block::mlp::moe::tests::common::helpers::{+ alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context,+};fn moe_cpu_reference(x: &[bf16],@@ -282,40 +284,39 @@ fn run_moe_parity_test_internal<B: Backend>(let down_biases: Vec<bf16> = (0..e * d_model).map(|_| bf16::from_f32(rng.random_range(-0.1..0.1))).collect();// Create Metal buffers- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);- let router_w_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_weight_bf16);- let router_b_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &router_bias_bf16);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13_gpu);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2_gpu);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);+ let router_w_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_weight_bf16);+ let router_b_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &router_bias_bf16);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13_gpu);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2_gpu);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);// Allocate intermediate buffers (max capacity)let max_sumk = t * k;- let mut topk_ids_buf = alloc_buffer::<B, i32>(&ctx, t * k);- let mut topk_probs_buf = alloc_buffer::<B, bf16>(&ctx, t * k);- let mut offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut sumk_buf = alloc_buffer::<B, u32>(&ctx, 1);+ let mut topk_ids_buf = alloc_allocation::<B, i32>(&ctx, t * k);+ let mut topk_probs_buf = alloc_allocation::<B, bf16>(&ctx, t * k);+ let mut offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut sumk_buf = alloc_allocation::<B, u32>(&ctx, 1);let num_blocks = ((t + 255) / 256).max(1);let num_tiles = ((e + 512 - 1) / 512).max(1);let entries = num_blocks * num_tiles * 512usize;- let mut partials_buf = alloc_buffer::<B, u32>(&ctx, entries);- let mut block_bases_buf = alloc_buffer::<B, u32>(&ctx, entries);- let mut block_alloc_buf = alloc_buffer::<B, u32>(&ctx, entries);- let mut bucketed_ids_buf = alloc_buffer::<B, i32>(&ctx, max_sumk);- let mut bucketed_probs_buf = alloc_buffer::<B, bf16>(&ctx, max_sumk);- let mut tok2row_buf = alloc_buffer::<B, i32>(&ctx, t * k);- let mut x_perm_buf = alloc_buffer::<B, bf16>(&ctx, max_sumk * d_model);- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, max_sumk * d_model);- let mut y_out_buf = alloc_buffer::<B, bf16>(&ctx, t * d_model);+ let mut partials_buf = alloc_allocation::<B, u32>(&ctx, entries);+ let mut block_bases_buf = alloc_allocation::<B, u32>(&ctx, entries);+ let mut block_alloc_buf = alloc_allocation::<B, u32>(&ctx, entries);+ let mut bucketed_ids_buf = alloc_allocation::<B, i32>(&ctx, max_sumk);+ let mut bucketed_probs_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk);+ let mut tok2row_buf = alloc_allocation::<B, i32>(&ctx, t * k);+ let mut x_perm_buf = alloc_allocation::<B, bf16>(&ctx, max_sumk * d_model);+ let mut y_out_buf = alloc_allocation::<B, bf16>(&ctx, t * d_model);const BLOCK_M_DECODE: usize = 4; // matches two-pass decode kernel configurationlet h_blocks_decode = (d_ff + BLOCK_M_DECODE - 1) / BLOCK_M_DECODE;let max_tiles = max_sumk * h_blocks_decode;- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_tiles * 3);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_tiles * 3);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);// Encode ALL kernels in one command buffereprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");@@ -386,29 +387,25 @@ fn run_moe_parity_test_internal<B: Backend>(&mut encoder,DataType::BF16,MoeGatherArguments {- x_buffer: &x_buf,- bucketed_ids_buffer: &bucketed_ids_buf,- x_perm_buffer: &mut x_perm_buf,- sumk_buffer: &sumk_buf,+ x: &x_buf,+ bucketed_ids: &bucketed_ids_buf,+ x_perm: &mut x_perm_buf,+ sumk: &sumk_buf,t,k,d_model,},);- // Additional buffers for 2-passlet total_rows = t * k;- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, total_rows * d_ff);- let mut row_expert_map_buf = alloc_buffer::<B, u32>(&ctx, total_rows);+ let mut row_expert_map_buf = alloc_allocation::<B, u32>(&ctx, total_rows);let experts = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("experts");let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;let args = MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,+ x_perm: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,@@ -431,7 +428,7 @@ fn run_moe_parity_test_internal<B: Backend>(silu_alpha,data_type: DataType::BF16,};- experts.encode(&mut encoder, args);+ let y_partial_buf = experts.encode(&mut encoder, args).expect("failed to encode MoE experts");let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");finalize.encode(@@ -446,11 +443,11 @@ fn run_moe_parity_test_internal<B: Backend>();eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();eprintln!("[E2E] GPU execution completed");// Read GPU output- let y_out_bf16 = unsafe { std::slice::from_raw_parts(y_out_buf.cpu_ptr().as_ptr() as *const bf16, t * d_model) };+ let y_out_bf16 = allocation_prefix_to_vec::<B, bf16>(&y_out_buf, t * d_model);let y_out_gpu: Vec<f32> = y_out_bf16.iter().map(|&v| f32::from(v)).collect();// Validate GPU output is finite@@ -475,13 +472,10 @@ fn run_moe_parity_test_internal<B: Backend>();// Probe tile bookkeeping- let total_tiles_cpu =- unsafe { std::slice::from_raw_parts(total_tiles_buf.cpu_ptr().as_ptr() as *const u32, 2) };- let dispatch_args_cpu =- unsafe { std::slice::from_raw_parts(dispatch_args_buf.cpu_ptr().as_ptr() as *const u32, 3) };- let tile_offsets_cpu =- unsafe { std::slice::from_raw_parts(tile_offsets_buf.cpu_ptr().as_ptr() as *const u32, (e + 1).min(8)) };- let sumk_val = unsafe { *(sumk_buf.cpu_ptr().as_ptr() as *const u32) } as usize;+ let total_tiles_cpu = allocation_prefix_to_vec::<B, u32>(&total_tiles_buf, 2);+ let dispatch_args_cpu = allocation_prefix_to_vec::<B, u32>(&dispatch_args_buf, 3);+ let tile_offsets_cpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, (e + 1).min(8));+ let sumk_val = allocation_to_vec::<B, u32>(&sumk_buf)[0] as usize;eprintln!("[E2E] Tile bookkeeping:");eprintln!("[E2E] total_tiles={}, dispatch_args=({}, {}, {})",@@ -492,8 +486,7 @@ fn run_moe_parity_test_internal<B: Backend>(// For multi-token tests with large d_ff, verify gather output (x_perm)if t > 1 && d_ff >= 256 {- let x_perm_cpu =- unsafe { std::slice::from_raw_parts(x_perm_buf.cpu_ptr().as_ptr() as *const bf16, sumk_val * d_model) };+ let x_perm_cpu = allocation_prefix_to_vec::<B, bf16>(&x_perm_buf, sumk_val * d_model);eprintln!("[E2E] x_perm diagnostics (sumk={}):", sumk_val);eprintln!("[E2E] Row 0 [0:8]: {:?}", &x_perm_cpu[0..8].iter().map(|&v| f32::from(v)).collect::<Vec<_>>());if sumk_val > 1 {@@ -506,14 +499,11 @@ fn run_moe_parity_test_internal<B: Backend>(}// Check tile_map for first few tiles- let tile_map_cpu = unsafe {- std::slice::from_raw_parts(tile_map_buf.cpu_ptr().as_ptr() as *const u32, 12.min(max_tiles * 3))- };+ let tile_map_cpu = allocation_prefix_to_vec::<B, u32>(&tile_map_buf, 12.min(max_tiles * 3));eprintln!("[E2E] tile_map (first 4 tiles): {:?}", &tile_map_cpu);// CRITICAL: Check tok2row mapping for multi-token tests- let tok2row_cpu =- unsafe { std::slice::from_raw_parts(tok2row_buf.cpu_ptr().as_ptr() as *const i32, t * k) };+ let tok2row_cpu = allocation_prefix_to_vec::<B, i32>(&tok2row_buf, t * k);eprintln!("[E2E] tok2row[0..{}]: {:?}", t * k, &tok2row_cpu);eprintln!("[E2E] Expected: token 0→row {}, token 1→row {}",@@ -526,9 +516,7 @@ fn run_moe_parity_test_internal<B: Backend>();// Check y_partial at specific indices where finalize reads for token 1- let y_partial_full = unsafe {- std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sumk_val * d_model)- };+ let y_partial_full = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sumk_val * d_model);eprintln!("[E2E] y_partial spot check:");eprintln!("[E2E] y_partial[48] (row 0, col 48) = {:.6}", f32::from(y_partial_full[48]));if sumk_val > 1 {@@ -612,14 +600,13 @@ fn run_moe_parity_test_internal<B: Backend>(// Debug: print some sample values from intermediate bufferseprintln!("[E2E] === DEBUG: Intermediate values ===");- let topk_ids_gpu = unsafe { std::slice::from_raw_parts(topk_ids_buf.cpu_ptr().as_ptr() as *const i32, t * k) };- let topk_probs_gpu = unsafe { std::slice::from_raw_parts(topk_probs_buf.cpu_ptr().as_ptr() as *const bf16, t * k) };+ let topk_ids_gpu = allocation_prefix_to_vec::<B, i32>(&topk_ids_buf, t * k);+ let topk_probs_gpu = allocation_prefix_to_vec::<B, bf16>(&topk_probs_buf, t * k);eprintln!("[E2E] TopK IDs: {:?}", topk_ids_gpu);eprintln!("[E2E] TopK Probs: {:?}", topk_probs_gpu.iter().map(|&v| f32::from(v)).collect::<Vec<_>>());- let y_partial_gpu =- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, max_sumk * d_model) };- let sumk_actual = unsafe { *(sumk_buf.cpu_ptr().as_ptr() as *const u32) } as usize;+ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, max_sumk * d_model);+ let sumk_actual = allocation_to_vec::<B, u32>(&sumk_buf)[0] as usize;eprintln!("[E2E] sumk={}", sumk_actual);let sample_size = 16.min(sumk_actual * d_model);eprintln!(@@ -682,6 +669,8 @@ fn run_moe_parity_test_internal<B: Backend>(}eprintln!("[{}] ✓ PASSED (GPU matches CPU reference)", test_name);+ drop(y_partial_buf);+ drop(completed);}// Test 1: Minimal (K=1, small dims, no clipping, alpha=1.0)diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rsindex 41b4a9ae1..41dd6cfec 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_experts_test.rs@@ -9,7 +9,7 @@use backend_uzu::{DataType,backends::common::{- DenseBuffer, Encoder,+ Encoder,gpu_types::{ActivationType, activation_silu_alpha},kernel::moe::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,@@ -23,7 +23,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::encodable_block::mlp::moe::tests::{common::{assert::assert_eq_float,- helpers::{alloc_buffer, alloc_buffer_with_data, create_context},+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, create_context},},cpu_tile_counts, cpu_tile_scan,};@@ -378,26 +378,23 @@ fn test_two_pass_decode_correctness() {);// Prepare GPU buffers- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x_perm);- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);- let mut row_expert_map_buf = alloc_buffer_with_data::<B, u32>(&ctx, &row_expert_map);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);-- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x_perm);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);+ let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &row_expert_map);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);// Tile infrastructurelet h_blocks_decode = (d_ff + 3) / 4;let max_total_tiles = sum_k * h_blocks_decode;- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_total_tiles * 3);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);// Execute 2-pass decode kernellet experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");@@ -406,43 +403,42 @@ fn test_two_pass_decode_correctness() {const K_TILE: usize = 64;let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;- experts_kernel.encode(- &mut encoder,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut y_partial_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k,- gating_code,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha,- data_type: DataType::BF16,- },- );-- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let y_partial_buf = experts_kernel+ .encode(+ &mut encoder,+ MoeExpertsTwoPassArguments {+ x_perm: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &mut row_expert_map_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &mut tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ tile_map: &mut tile_map_buf,+ total_tiles: &mut total_tiles_buf,+ dispatch_args: &mut dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k,+ gating_code,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");++ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();// Read GPU partial output and do CPU finalize (weighted sum)- let y_partial_gpu =- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sum_k * d_model) };+ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sum_k * d_model);// Finalize: y[t] = Σ_k prob[t,k] * y_partial[t*k + k_idx]let mut y_gpu = vec![bf16::from_f32(0.0); t * d_model];@@ -483,6 +479,8 @@ fn test_two_pass_decode_correctness() {assert_eq_float(&y_expected, &y_gpu, tolerance, "2-pass decode output");eprintln!("[2-pass decode] ✓ PASSED (tolerance={:.4})", tolerance);+ drop(y_partial_buf);+ drop(completed);});}@@ -527,65 +525,63 @@ fn test_two_pass_decode_multi_token() {);// GPU buffers- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);- let mut row_expert_map_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.row_expert_map);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w13);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w2);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);-+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);+ let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);// Tile infrastructurelet max_total_tiles = sum_k * ((d_ff + 3) / 4);- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_total_tiles * 3);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassDecodeBlock::<B>::new(&ctx).expect("kernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- experts_kernel.encode(- &mut encoder,- MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut y_partial_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &mut tile_counts_buf,- tile_offsets: &mut tile_offsets_buf,- tile_map: &mut tile_map_buf,- total_tiles: &mut total_tiles_buf,- dispatch_args: &mut dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: ((d_ff + 63) / 64) as u32,- gating_code,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha,- data_type: DataType::BF16,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let y_partial_gpu =- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sum_k * d_model) };- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);+ let y_partial_buf = experts_kernel+ .encode(+ &mut encoder,+ MoeExpertsTwoPassArguments {+ x_perm: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &mut row_expert_map_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &mut tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ tile_map: &mut tile_map_buf,+ total_tiles: &mut total_tiles_buf,+ dispatch_args: &mut dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: ((d_ff + 63) / 64) as u32,+ gating_code,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();++ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sum_k * d_model);+ let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass decode multi-token");eprintln!("[2-pass decode multi-token] ✓ PASSED");+ drop(y_partial_buf);+ drop(completed);});}@@ -630,32 +626,27 @@ fn test_two_pass_prefill_correctness() {);// GPU buffers- let x_perm_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &scatter.x_perm);- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.offsets);- let mut row_expert_map_buf = alloc_buffer_with_data::<B, u32>(&ctx, &scatter.row_expert_map);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w13);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.w2);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &data.down_biases);- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, sum_k * d_ff);- let mut y_partial_buf = alloc_buffer::<B, bf16>(&ctx, sum_k * d_model);-+ let x_perm_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &scatter.x_perm);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.offsets);+ let mut row_expert_map_buf = alloc_allocation_with_data::<B, u32>(&ctx, &scatter.row_expert_map);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w13);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.w2);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &data.down_biases);// Tile infrastructurelet max_total_tiles = sum_k * ((d_ff + 3) / 4);- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 8);- let mut tile_map_buf = alloc_buffer::<B, u32>(&ctx, max_total_tiles * 3);- let mut dispatch_args_buf = alloc_buffer::<B, u32>(&ctx, 3);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 8);+ let mut tile_map_buf = alloc_allocation::<B, u32>(&ctx, max_total_tiles * 3);+ let mut dispatch_args_buf = alloc_allocation::<B, u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassPrefillBlock::<B>::new(&ctx).expect("kernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,+ x_perm: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &mut row_expert_map_buf,- hidden_buffer: &mut hidden_buf,- output_buffer: &mut y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,up_biases: &up_biases_buf,@@ -678,15 +669,16 @@ fn test_two_pass_prefill_correctness() {silu_alpha,data_type: DataType::BF16,};- experts_kernel.encode(&mut encoder, args);- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let y_partial_buf = experts_kernel.encode(&mut encoder, args).expect("failed to encode MoE experts");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();- let y_partial_gpu =- unsafe { std::slice::from_raw_parts(y_partial_buf.cpu_ptr().as_ptr() as *const bf16, sum_k * d_model) };- let y_gpu = gather_and_finalize(y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);+ let y_partial_gpu = allocation_prefix_to_vec::<B, bf16>(&y_partial_buf, sum_k * d_model);+ let y_gpu = gather_and_finalize(&y_partial_gpu, &data.topk_probs, &scatter.perm_idx, t, k, d_model);assert_eq_float(&y_expected, &y_gpu, 0.02, "2-pass prefill");eprintln!("[2-pass prefill] ✓ PASSED");+ drop(y_partial_buf);+ drop(completed);});}@@ -768,49 +760,46 @@ fn test_fused_single_token_decode() {);// GPU buffers- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(&ctx, &topk_ids);- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &topk_probs);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13_all);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2_all);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, k * d_ff);- let mut y_buf = alloc_buffer::<B, bf16>(&ctx, d_model);-+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(&ctx, &topk_ids);+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &topk_probs);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13_all);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2_all);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);// Run fused decode kernellet fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- fused_kernel.encode(- &mut encoder,- MoeExpertsSingleDecodeArguments {- x: &x_buf,- topk_ids: &topk_ids_buf,- topk_probs: &topk_probs_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- hidden: &mut hidden_buf,- y: &mut y_buf,- d_model,- d_ff,- k,- gating_code,- silu_alpha: 1.0, // Standard SiLU for testing- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- data_type: DataType::BF16,- },- );-- encoder.end_encoding().submit().wait_until_completed().unwrap();+ let y_buf = fused_kernel+ .encode(+ &mut encoder,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ d_model,+ d_ff,+ k,+ gating_code,+ silu_alpha: 1.0, // Standard SiLU for testing+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");++ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();// Read GPU output- let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr().as_ptr() as *const bf16, d_model) };+ let y_gpu = allocation_prefix_to_vec::<B, bf16>(&y_buf, d_model);// Compute error metricslet mut max_abs_error = 0.0f32;@@ -837,9 +826,11 @@ fn test_fused_single_token_decode() {);let tolerance = 0.02;- assert_eq_float(&y_expected, y_gpu, tolerance, "fused single-token output");+ assert_eq_float(&y_expected, &y_gpu, tolerance, "fused single-token output");eprintln!("[fused single-token] PASSED (tolerance={:.4})", tolerance);+ drop(y_buf);+ drop(completed);});}@@ -895,45 +886,42 @@ fn test_fused_single_token_k4() {f32::INFINITY,);- let x_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &x);- let topk_ids_buf = alloc_buffer_with_data::<B, i32>(&ctx, &topk_ids);- let topk_probs_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &topk_probs);- let w13_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w13_all);- let w2_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &w2_all);- let up_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &up_biases);- let down_biases_buf = alloc_buffer_with_data::<B, bf16>(&ctx, &down_biases);- let mut hidden_buf = alloc_buffer::<B, f32>(&ctx, k * d_ff);- let mut y_buf = alloc_buffer::<B, bf16>(&ctx, d_model);-+ let x_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &x);+ let topk_ids_buf = alloc_allocation_with_data::<B, i32>(&ctx, &topk_ids);+ let topk_probs_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &topk_probs);+ let w13_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w13_all);+ let w2_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &w2_all);+ let up_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &up_biases);+ let down_biases_buf = alloc_allocation_with_data::<B, bf16>(&ctx, &down_biases);let fused_kernel = MoeExpertsSingleDecodeKernels::<B>::new(&ctx).expect("fused kernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");- fused_kernel.encode(- &mut encoder,- MoeExpertsSingleDecodeArguments {- x: &x_buf,- topk_ids: &topk_ids_buf,- topk_probs: &topk_probs_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- hidden: &mut hidden_buf,- y: &mut y_buf,- d_model,- d_ff,- k,- gating_code,- silu_alpha,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- data_type: DataType::BF16,- },- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.cpu_ptr().as_ptr() as *const bf16, d_model) };+ let y_buf = fused_kernel+ .encode(+ &mut encoder,+ MoeExpertsSingleDecodeArguments {+ x: &x_buf,+ topk_ids: &topk_ids_buf,+ topk_probs: &topk_probs_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ d_model,+ d_ff,+ k,+ gating_code,+ silu_alpha,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ data_type: DataType::BF16,+ },+ )+ .expect("failed to encode MoE experts");+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();++ let y_gpu = allocation_prefix_to_vec::<B, bf16>(&y_buf, d_model);// Compute error metricslet mut max_abs_error = 0.0f32;@@ -960,8 +948,10 @@ fn test_fused_single_token_k4() {);let tolerance = 0.02;- assert_eq_float(y_gpu, &y_expected, tolerance, "fused single-token K=4 output");+ assert_eq_float(&y_gpu, &y_expected, tolerance, "fused single-token K=4 output");eprintln!("[fused single-token K=4] PASSED (tolerance={:.4})", tolerance);+ drop(y_buf);+ drop(completed);});}diff --git a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rsindex fd9190ac2..85b4de671 100644--- a/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/moe/moe_tiles_test.rs@@ -1,11 +1,13 @@use backend_uzu::backends::common::{- DenseBuffer, Encoder,+ Encoder,kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},};use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::encodable_block::mlp::moe::tests::{- common::helpers::{alloc_buffer, alloc_buffer_with_data, create_context},+ common::helpers::{+ alloc_allocation, alloc_allocation_with_data, allocation_prefix_to_vec, allocation_to_vec, create_context,+ },cpu_tile_counts, cpu_tile_scan,};@@ -31,8 +33,8 @@ fn test_tile_counts_correctness() {let tile_counts_cpu = cpu_tile_counts(&offsets, 16);// GPU buffers- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");@@ -40,18 +42,16 @@ fn test_tile_counts_correctness() {tile_kernel.encode_counts(&mut encoder,MoeTileCountsArguments {- offsets_buffer: &offsets_buf,- tile_counts_buffer: &mut tile_counts_buf,+ offsets: &offsets_buf,+ tile_counts: &mut tile_counts_buf,e,},);encoder.end_encoding().submit().wait_until_completed().unwrap();- // Compare- let tile_counts_gpu =- unsafe { std::slice::from_raw_parts(tile_counts_buf.cpu_ptr().as_ptr() as *const u32, e) };+ let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);- assert_eq!(tile_counts_gpu, &tile_counts_cpu[..], "Tile counts mismatch for E={}", e);+ assert_eq!(tile_counts_gpu, tile_counts_cpu, "Tile counts mismatch for E={}", e);eprintln!("[TileCountsTest] ✓ PASSED");}@@ -76,9 +76,9 @@ fn test_tile_scan_correctness() {let (tile_offsets_cpu, total_tiles_cpu) = cpu_tile_scan(&tile_counts);// GPU buffers- let tile_counts_buf = alloc_buffer_with_data::<B, u32>(&ctx, &tile_counts);- let mut tile_offsets_buf = alloc_buffer::<B, u32>(&ctx, e + 1);- let mut total_tiles_buf = alloc_buffer::<B, u32>(&ctx, 1);+ let tile_counts_buf = alloc_allocation_with_data::<B, u32>(&ctx, &tile_counts);+ let mut tile_offsets_buf = alloc_allocation::<B, u32>(&ctx, e + 1);+ let mut total_tiles_buf = alloc_allocation::<B, u32>(&ctx, 1);// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");@@ -86,20 +86,18 @@ fn test_tile_scan_correctness() {tile_kernel.encode_scan(&mut encoder,MoeTileScanArguments {- tile_counts_buffer: &tile_counts_buf,- tile_offsets_buffer: &mut tile_offsets_buf,- total_tiles_buffer: &mut total_tiles_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &mut tile_offsets_buf,+ total_tiles: &mut total_tiles_buf,e,},);encoder.end_encoding().submit().wait_until_completed().unwrap();- // Compare- let tile_offsets_gpu =- unsafe { std::slice::from_raw_parts(tile_offsets_buf.cpu_ptr().as_ptr() as *const u32, e + 1) };- let total_tiles_gpu = unsafe { *(total_tiles_buf.cpu_ptr().as_ptr() as *const u32) };+ let tile_offsets_gpu = allocation_prefix_to_vec::<B, u32>(&tile_offsets_buf, e + 1);+ let total_tiles_gpu = allocation_to_vec::<B, u32>(&total_tiles_buf)[0];- assert_eq!(tile_offsets_gpu, &tile_offsets_cpu[..], "Tile offsets mismatch for E={}", e);+ assert_eq!(tile_offsets_gpu, tile_offsets_cpu, "Tile offsets mismatch for E={}", e);assert_eq!(total_tiles_gpu, total_tiles_cpu, "Total tiles mismatch for E={}", e);eprintln!("[TileScanTest] ✓ PASSED");@@ -119,25 +117,24 @@ fn test_tile_edge_cases() {let tile_counts_cpu = cpu_tile_counts(&offsets, 16);- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernel::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");tile_kernel.encode_counts(&mut encoder,MoeTileCountsArguments {- offsets_buffer: &offsets_buf,- tile_counts_buffer: &mut tile_counts_buf,+ offsets: &offsets_buf,+ tile_counts: &mut tile_counts_buf,e,},);encoder.end_encoding().submit().wait_until_completed().unwrap();- let tile_counts_gpu =- unsafe { std::slice::from_raw_parts(tile_counts_buf.cpu_ptr().as_ptr() as *const u32, e) };+ let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);- assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);+ assert_eq!(tile_counts_gpu, tile_counts_cpu);assert!(tile_counts_gpu.iter().all(|&c| c == 0));eprintln!("[TileEdgeCases] ✓ Empty experts PASSED");}@@ -149,25 +146,24 @@ fn test_tile_edge_cases() {let tile_counts_cpu = cpu_tile_counts(&offsets, 16);- let offsets_buf = alloc_buffer_with_data::<B, u32>(&ctx, &offsets);- let mut tile_counts_buf = alloc_buffer::<B, u32>(&ctx, e);+ let offsets_buf = alloc_allocation_with_data::<B, u32>(&ctx, &offsets);+ let mut tile_counts_buf = alloc_allocation::<B, u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<B>::new(&ctx).expect("MoeTileMapKernels::new");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");tile_kernel.encode_counts(&mut encoder,MoeTileCountsArguments {- offsets_buffer: &offsets_buf,- tile_counts_buffer: &mut tile_counts_buf,+ offsets: &offsets_buf,+ tile_counts: &mut tile_counts_buf,e,},);encoder.end_encoding().submit().wait_until_completed().unwrap();- let tile_counts_gpu =- unsafe { std::slice::from_raw_parts(tile_counts_buf.cpu_ptr().as_ptr() as *const u32, e) };+ let tile_counts_gpu = allocation_prefix_to_vec::<B, u32>(&tile_counts_buf, e);- assert_eq!(tile_counts_gpu, &tile_counts_cpu[..]);+ assert_eq!(tile_counts_gpu, tile_counts_cpu);// With BM=16: seg_lens=[10, 5, 15, 20] -> tile_counts=[1, 1, 1, 2]assert_eq!(tile_counts_gpu, &[1, 1, 1, 2]);eprintln!("[TileEdgeCases] ✓ Small segments PASSED");diff --git a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rsindex 723749022..f52078d55 100644--- a/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs+++ b/crates/backend-uzu/tests/unit/encodable_block/sampling_test.rs@@ -1,7 +1,5 @@#![cfg(metal_backend)]-use bytemuck;-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use rand::seq::SliceRandom;// for Vec::shuffle@@ -20,6 +18,11 @@ use crate::{session::parameter::{SamplingMethod, SamplingProcessingOrder},};+#[path = "../../common/mod.rs"]+mod common;++use common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec};+// Constant seed for reproducible test resultsconst TEST_SAMPLING_SEED: u64 = 42;@@ -54,7 +57,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {let batch_size = 2;let vocab_size = 4;- let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, batch_size, vocab_size, strategy)+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, strategy).expect("Failed to create argmax kernel");// Create test data: batch_size=2, vocab_size=4@@ -65,29 +68,16 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {0.1, 0.5, 2.5, 1.0, // batch 1];- let mut logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let seeds_buffer = context- .device- .new_buffer_with_data(&vec![0; 16], MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut output_buffer = context- .device- .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &test_logits);+ let seeds_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &[0_u64, 0_u64]);+ let mut output_buffer = alloc_allocation::<Metal, u32>(context.as_ref(), batch_size);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(&mut logits_buffer,&seeds_buffer,- 0,None,- 0,&mut output_buffer,SamplingMethod::Greedy,batch_size,@@ -98,8 +88,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {encoder.end_encoding().submit().wait_until_completed().unwrap();// Check results- let result_ptr = output_buffer.contents().as_ptr() as *const u32;- let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };+ let results: Vec<u32> = allocation_to_vec(&output_buffer);assert_eq!(results[0], 1, "First batch should select token 1 (highest logit: 3.0)");assert_eq!(results[1], 2, "Second batch should select token 2 (highest logit: 2.5)");@@ -123,7 +112,6 @@ fn test_argmax_sampling() {test_argmax_sampling_with_strategy(ArgmaxStrategy::SinglePass);}-#[allow(dead_code)]fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {use std::time::Instant;@@ -143,7 +131,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {const VOCAB: usize = 128000; // 128K// ---- Kernel ----- let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, BATCH, VOCAB, strategy)+ let kernel = SamplingKernel::<Metal>::new_with_strategy(&context, DataType::F32, strategy).expect("Failed to create Argmax kernel");// ---- Build random logits ----@@ -153,21 +141,12 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {*x = rng.random_range(-6.0f32..6.0f32);}- let mut logits_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut logits_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];- let seeds_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let seeds_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds);- let mut output_buf = context- .device- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut output_buf = alloc_allocation::<Metal, u32>(context.as_ref(), BATCH);// ---- Launch once and time ----let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -176,9 +155,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {.encode(&mut logits_buf,&seeds_buf,- 0,- None,- 0,+ None::<&backend_uzu::backends::common::Allocation<Metal>>,&mut output_buf,SamplingMethod::Greedy,BATCH,@@ -199,9 +176,8 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {);// Ensure the kernel produced *some* output (sanity).- let ptr = output_buf.contents().as_ptr() as *const u32;- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };- for &tok in sample_ids {+ let sample_ids: Vec<u32> = allocation_to_vec(&output_buf);+ for &tok in &sample_ids {assert!((tok as usize) < VOCAB, "Sampled id out of range");}@@ -227,6 +203,18 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {println!("✓ Argmax correctness verified with {:?} strategy", strategy);}+#[test]+#[ignore = "performance-only check"]+fn perf_argmax_128k_vocab_single_pass() {+ perf_argmax_128k_vocab_with_strategy(ArgmaxStrategy::SinglePass);+}++#[test]+#[ignore = "performance-only check"]+fn perf_argmax_128k_vocab_two_pass() {+ perf_argmax_128k_vocab_with_strategy(ArgmaxStrategy::TwoPass);+}+#[test]fn test_categorical_sampling() {let context = match <Metal as Backend>::Context::new() {@@ -240,8 +228,7 @@ fn test_categorical_sampling() {let batch_size = 2;let vocab_size = 4;- let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)- .expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");// Create test data with different probability distributions// First batch: [1.0, 2.0, 1.5, 0.5] -> softmax: [0.134, 0.366, 0.201, 0.082]@@ -251,10 +238,7 @@ fn test_categorical_sampling() {0.0, 1.0, 0.0, 0.0, // batch 1];- let mut output_buffer = context- .device- .new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut output_buffer = alloc_allocation::<Metal, u32>(context.as_ref(), batch_size);// Run sampling multiple times to check distributionlet num_samples = 1000;@@ -262,16 +246,10 @@ fn test_categorical_sampling() {for sample_idx in 0..num_samples {// Create fresh logits buffer since kernel mutates in-place- let mut logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &test_logits);let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; batch_size];- let seeds_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let seeds_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds_vec);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -279,9 +257,7 @@ fn test_categorical_sampling() {.encode(&mut logits_buffer,&seeds_buffer,- 0,None,- 0,&mut output_buffer,SamplingMethod::Stochastic {temperature: None,@@ -297,8 +273,7 @@ fn test_categorical_sampling() {.expect("Categorical sampling should succeed");encoder.end_encoding().submit().wait_until_completed().unwrap();- let result_ptr = output_buffer.contents().as_ptr() as *const u32;- let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };+ let results: Vec<u32> = allocation_to_vec(&output_buffer);for (batch_idx, &token) in results.iter().enumerate() {assert!(@@ -360,8 +335,7 @@ fn test_categorical_sampling_statistical() {const NUM_SAMPLES: usize = 5000;const TOLERANCE: f32 = 0.05; // 5% tolerance- let kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");// Generate random logitslet mut rng = StdRng::seed_from_u64(42);@@ -389,26 +363,17 @@ fn test_categorical_sampling_statistical() {}}- let mut output_buffer = context- .device- .new_buffer(BATCH * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut output_buffer = alloc_allocation::<Metal, u32>(context.as_ref(), BATCH);let mut counts = vec![0; BATCH * VOCAB];// Sample many timesfor sample_idx in 0..NUM_SAMPLES {// Create fresh logits buffer since kernel mutates in-place- let mut logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);let seeds_vec = vec![TEST_SAMPLING_SEED + sample_idx as u64; BATCH];- let seeds_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let seeds_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds_vec);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -416,9 +381,7 @@ fn test_categorical_sampling_statistical() {.encode(&mut logits_buffer,&seeds_buffer,- 0,None,- 0,&mut output_buffer,SamplingMethod::Stochastic {temperature: None,@@ -434,8 +397,7 @@ fn test_categorical_sampling_statistical() {.expect("encode");encoder.end_encoding().submit().wait_until_completed().unwrap();- let result_ptr = output_buffer.contents().as_ptr() as *const u32;- let results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH) };+ let results: Vec<u32> = allocation_to_vec(&output_buffer);for (batch_idx, &token) in results.iter().enumerate() {counts[batch_idx * VOCAB + token as usize] += 1;@@ -485,8 +447,7 @@ fn perf_categorical_128k_vocab() {const BATCH: usize = 8;const VOCAB: usize = 128000; // 128K- let kernel =- SamplingKernel::<Metal>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");// Build random logitslet mut rng = StdRng::seed_from_u64(123);@@ -495,21 +456,12 @@ fn perf_categorical_128k_vocab() {*x = rng.random_range(-6.0f32..6.0f32);}- let mut logits_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut logits_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];- let seeds_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let seeds_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds);- let mut output_buf = context- .device- .new_buffer(BATCH * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut output_buf = alloc_allocation::<Metal, u32>(context.as_ref(), BATCH);// Launch and timelet mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -518,9 +470,7 @@ fn perf_categorical_128k_vocab() {.encode(&mut logits_buf,&seeds_buf,- 0,None,- 0,&mut output_buf,SamplingMethod::Stochastic {temperature: None,@@ -546,9 +496,8 @@ fn perf_categorical_128k_vocab() {);// Sanity check- let ptr = output_buf.contents().as_ptr() as *const u32;- let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };- for &tok in sample_ids {+ let sample_ids: Vec<u32> = allocation_to_vec(&output_buf);+ for &tok in &sample_ids {assert!((tok as usize) < VOCAB, "Sampled id out of range");}@@ -576,22 +525,14 @@ fn test_temperature_gpu_cpu_match() {let logits: Vec<f32> = (0..BATCH * VOCAB).map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01).collect();- let logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut processed_buffer = context- .device- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);+ let mut processed_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), logits.len());let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();- let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;- let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };+ let gpu_results: Vec<f32> = allocation_to_vec(&processed_buffer);for (idx, (&logit, &processed)) in logits.iter().zip(gpu_results.iter()).enumerate() {let expected = logit / TEMPERATURE;@@ -636,22 +577,14 @@ fn test_minp_gpu_cpu_match() {*x = rng.random_range(-16.0f32..16.0f32);}- let logits_buffer = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut processed_buffer = context- .device- .new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let logits_buffer = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);+ let mut processed_buffer = alloc_allocation::<Metal, f32>(context.as_ref(), logits.len());let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();- let results_ptr = processed_buffer.contents().as_ptr() as *const f32;- let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };+ let all_results: Vec<f32> = allocation_to_vec(&processed_buffer);for batch_idx in 0..BATCH {let cpu_logits = &logits[batch_idx * VOCAB..(batch_idx + 1) * VOCAB];@@ -682,8 +615,7 @@ fn test_minp_sampling_exact_match(},};- let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32, batch_size, vocab_size)- .expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<Metal>::new(&context, DataType::F32).expect("Failed to create sampling kernel");// Build logits where some tokens have high probability and others have low// For min_p filtering, tokens with probability < min_p * max_prob are masked@@ -708,35 +640,24 @@ fn test_minp_sampling_exact_match(high_prob_token_sets.push(high_prob_tokens);}- let mut output_buf = context- .device- .new_buffer(batch_size * std::mem::size_of::<u32>(), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut output_buf = alloc_allocation::<Metal, u32>(context.as_ref(), batch_size);let num_samples = 1000;let mut counter = vec![0i32; batch_size * vocab_size];for draw in 0..num_samples {// Create fresh logits buffer since kernel mutates in-place- let mut logits_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&logits), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let mut logits_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &logits);let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];- let seeds_buf = context- .device- .new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");+ let seeds_buf = alloc_allocation_with_data::<Metal, _>(context.as_ref(), &seeds);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(&mut logits_buf,&seeds_buf,- 0,None,- 0,&mut output_buf,SamplingMethod::Stochastic {temperature: None,@@ -752,8 +673,7 @@ fn test_minp_sampling_exact_match(.expect("encode");encoder.end_encoding().submit().wait_until_completed().unwrap();- let ptr = output_buf.contents().as_ptr() as *const u32;- let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };+ let sampled_ids: Vec<u32> = allocation_to_vec(&output_buf);for (i, &sampled_id) in sampled_ids.iter().enumerate() {assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");diff --git a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rsindex 3b60230f7..95a66311b 100644--- a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs+++ b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs@@ -1,14 +1,19 @@#![cfg(metal_backend)]use crate::{- ArrayContextExt, DataType,+ DataType,backends::{- common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},+ common::{Allocation, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},metal::Metal,},forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},};+#[path = "../../common/mod.rs"]+mod common;++use common::helpers::{alloc_allocation_with_data, allocation_to_vec, write_allocation};+#[derive(Debug)]struct Scenario {name: &'static str,@@ -40,31 +45,44 @@ fn make_test_layer(} => window_length + suffix_capacity,};let shape = [total_len.max(1), 1, 1];- let keys = context.create_array_zeros(&shape, DataType::F32, "kv_cache_keys");- let values = context.create_array_zeros(&shape, DataType::F32, "kv_cache_values");++ let zeroes = vec![0.0_f32; shape.iter().product()];+ let keys = alloc_allocation_with_data(context, &zeroes);+ let values = alloc_allocation_with_data(context, &zeroes);KVCacheLayer {state,keys,values,+ shape,+ data_type: DataType::F32,}}+fn overwrite_allocation(+ allocation: &mut Allocation<Metal>,+ updates: &[(usize, f32)],+) {+ let mut data: Vec<f32> = allocation_to_vec(allocation);+ for (index, value) in updates {+ data[*index] = *value;+ }+ write_allocation(allocation, &data);+}+fn fill_arrays(layer: &mut KVCacheLayer<Metal>) -> (Vec<f32>, Vec<f32>) {let initial_keys = {- let slice = layer.keys.as_slice_mut::<f32>();- for (idx, value) in slice.iter_mut().enumerate() {- *value = 1_000.0 + idx as f32;- }- slice.to_vec()+ let len = layer.shape.iter().product();+ let data: Vec<f32> = (0..len).map(|idx| 1_000.0 + idx as f32).collect();+ write_allocation(&mut layer.keys, &data);+ data};let initial_values = {- let slice = layer.values.as_slice_mut::<f32>();- for (idx, value) in slice.iter_mut().enumerate() {- *value = 2_000.0 + idx as f32;- }- slice.to_vec()+ let len = layer.shape.iter().product();+ let data: Vec<f32> = (0..len).map(|idx| 2_000.0 + idx as f32).collect();+ write_allocation(&mut layer.values, &data);+ data};(initial_keys, initial_values)@@ -154,10 +172,10 @@ fn run_scenario(layer.register_accepted_tokens(scenario.number_of_accepted_tokens);- let actual_keys = layer.keys.as_slice::<f32>().to_vec();+ let actual_keys: Vec<f32> = allocation_to_vec(&layer.keys);assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);- let actual_values = layer.values.as_slice::<f32>().to_vec();+ let actual_values: Vec<f32> = allocation_to_vec(&layer.values);assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);match &layer.state {@@ -290,17 +308,13 @@ fn kv_cache_slice_apply_contiguous_window() {let slice = layer.slice(&context, 0..2).expect("slice should exist");// Mutate the captured slots.- {- layer.keys.as_slice_mut::<f32>()[0] = -1.0;- layer.keys.as_slice_mut::<f32>()[1] = -2.0;- layer.values.as_slice_mut::<f32>()[0] = -3.0;- layer.values.as_slice_mut::<f32>()[1] = -4.0;- }+ overwrite_allocation(&mut layer.keys, &[(0, -1.0), (1, -2.0)]);+ overwrite_allocation(&mut layer.values, &[(0, -3.0), (1, -4.0)]);layer.apply_slice(&slice, None);- let keys_after = layer.keys.as_slice::<f32>().to_vec();- let values_after = layer.values.as_slice::<f32>().to_vec();+ let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);+ let values_after: Vec<f32> = allocation_to_vec(&layer.values);assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");assert_eq!(values_after[0..4], initial_values[0..4], "values restored for contiguous slice");}@@ -325,17 +339,13 @@ fn kv_cache_slice_apply_wrap_window() {let slice = layer.slice(&context, 2..4).expect("slice should exist");// Captured slots are expected to wrap; mutate them.- {- layer.keys.as_slice_mut::<f32>()[2] = -11.0;- layer.keys.as_slice_mut::<f32>()[3] = -12.0;- layer.values.as_slice_mut::<f32>()[2] = -13.0;- layer.values.as_slice_mut::<f32>()[3] = -14.0;- }+ overwrite_allocation(&mut layer.keys, &[(2, -11.0), (3, -12.0)]);+ overwrite_allocation(&mut layer.values, &[(2, -13.0), (3, -14.0)]);layer.apply_slice(&slice, None);- let keys_after = layer.keys.as_slice::<f32>().to_vec();- let values_after = layer.values.as_slice::<f32>().to_vec();+ let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);+ let values_after: Vec<f32> = allocation_to_vec(&layer.values);assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");assert_eq!(values_after[0..4], initial_values[0..4], "values restored for wrapped slice");}diff --git a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rsindex ca591a5b0..db5e5ea56 100644--- a/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/activation_mul_test.rs@@ -1,19 +1,22 @@-use std::{- fmt::{Debug, Display},- ops::{Deref, DerefMut},-};+use std::fmt::{Debug, Display};+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{- ArrayContextExt, ArrayElement, DataType,+ ArrayElement, DataType,backends::{common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel},cpu::Cpu,},};-use half::{bf16, f16};-use num_traits::Float;-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+ uzu_test,+};struct Input<T: ArrayElement + Float> {fused_up: Box<[T]>,@@ -52,14 +55,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let fused_len = (input.m * 2 * input.h) as usize;let out_len = (input.m * input.h) as usize;- let fused_up_array = context.create_array_from(&[fused_len], &input.fused_up, "");- let hidden_array = context.create_array_uninitialized(&[out_len], T::data_type(), "");+ let fused_up_allocation = alloc_allocation_with_data::<B, T>(&context, &input.fused_up[..fused_len]);+ let mut hidden_allocation = alloc_allocation::<B, T>(&context, out_len);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- fused_up_array.buffer().borrow().deref(),- hidden_array.buffer().borrow_mut().deref_mut(),- None::<&B::DenseBuffer>,+ &fused_up_allocation,+ &mut hidden_allocation,+ None::<&backend_uzu::backends::common::Allocation<B>>,input.h,input.m,input.act_type,@@ -67,7 +70,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- hidden_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&hidden_allocation)}fn test<T: ArrayElement + Float + Debug + Display>(act_type: ActivationType) {diff --git a/crates/backend-uzu/tests/unit/kernel/activation_test.rs b/crates/backend-uzu/tests/unit/kernel/activation_test.rsindex 9faed48a9..32466262e 100644--- a/crates/backend-uzu/tests/unit/kernel/activation_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/activation_test.rs@@ -1,19 +1,22 @@-use std::{- fmt::{Debug, Display},- ops::{Deref, DerefMut},-};+use std::fmt::{Debug, Display};+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{- ArrayContextExt, ArrayElement, DataType,+ ArrayElement, DataType,backends::{common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::ActivationKernel},cpu::Cpu,},};-use half::{bf16, f16};-use num_traits::Float;-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+ uzu_test,+};struct Input<T: ArrayElement + Float> {data: Box<[T]>,@@ -38,30 +41,24 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let n = input.data.len();if input.in_place {- let output_array = context.create_array_from(&[n], &input.data, "");+ let mut output_allocation = alloc_allocation_with_data::<B, T>(&context, &input.data);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");kernel.encode(- Option::<&B::DenseBuffer>::None,- output_array.buffer().borrow_mut().deref_mut(),+ None::<&backend_uzu::backends::common::Allocation<B>>,+ &mut output_allocation,n as u32,input.act_type,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&output_allocation)} else {- let input_array = context.create_array_from(&[n], &input.data, "");- let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");+ let input_allocation = alloc_allocation_with_data::<B, T>(&context, &input.data);+ let mut output_allocation = alloc_allocation::<B, T>(&context, n);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");- kernel.encode(- Some(input_array.buffer().borrow().deref()),- output_array.buffer().borrow_mut().deref_mut(),- n as u32,- input.act_type,- &mut encoder,- );+ kernel.encode(Some(&input_allocation), &mut output_allocation, n as u32, input.act_type, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&output_allocation)}}diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rsindex 250114a25..b1dff7fde 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs@@ -1,19 +1,25 @@-use std::{- fmt::{Debug, Display},- ops::{Deref, DerefMut},-};+use std::fmt::{Debug, Display};use backend_uzu::{- ArrayContextExt, ArrayElement, DataType,+ ArrayElement, DataType,backends::{- common::{Backend, Context, Encoder, kernel::attention::AttentionGemmArguments},+ common::{+ Backend, Context, Encoder,+ kernel::attention::{AttentionGemmArguments, AttentionGemmBlock},+ },cpu::Cpu,},};use half::{bf16, f16};use num_traits::Float;-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+ uzu_test,+};struct Input<T: ArrayElement + Float> {queries: Box<[T]>,@@ -79,57 +85,46 @@ fn get_test_data<T: ArrayElement + Float>(fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let block = backend_uzu::backends::common::kernel::attention::AttentionGemmBlock::<B>::new(T::data_type());+ let block = AttentionGemmBlock::<B>::new(T::data_type());- let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");- let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");- let values_array = context.create_array_from(&[input.values.len()], &input.values, "");+ let queries_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.queries);+ let keys_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.keys);+ let values_allocation = alloc_allocation_with_data::<B, T>(context.as_ref(), &input.values);let output_size = input.suffix_length * input.num_heads * input.head_dim;- let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");+ let mut output_allocation = alloc_allocation::<B, T>(context.as_ref(), output_size);let segment_prefix_length = input.sequence_length - input.suffix_length;- {- let q_buf_rc = queries_array.buffer();- let q_buf = q_buf_rc.borrow();- let k_buf_rc = keys_array.buffer();- let k_buf = k_buf_rc.borrow();- let v_buf_rc = values_array.buffer();- let v_buf = v_buf_rc.borrow();- let o_buf_rc = output_array.buffer();- let mut o_buf = o_buf_rc.borrow_mut();-- let args = AttentionGemmArguments::<B> {- queries_buffer: q_buf.deref(),- keys_buffer: k_buf.deref(),- values_buffer: v_buf.deref(),- output_buffer: o_buf.deref_mut(),- trie_buffer: None,- sinks_buffer: None,- num_heads: input.num_heads,- num_groups: input.num_kv_heads,- suffix_length: input.suffix_length,- sequence_length: input.sequence_length,- segment_prefix_length,- max_sequence_length: input.sequence_length,- ring_params: None,- head_dim: input.head_dim,- sliding_window_size: None,- is_causal: input.do_causal,- scale: input.scale,- k_head_stride: input.head_dim as u64,- k_seq_stride: (input.num_kv_heads * input.head_dim) as u64,- v_head_stride: input.head_dim as u64,- v_seq_stride: (input.num_kv_heads * input.head_dim) as u64,- };-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- block.encode(&context, &mut encoder, args).expect("Failed to encode AttentionGemm");- encoder.end_encoding().submit().wait_until_completed().unwrap();- }+ let args = AttentionGemmArguments::<B> {+ queries: &queries_allocation,+ keys: &keys_allocation,+ values: &values_allocation,+ output: &mut output_allocation,+ trie: None,+ sinks: None,+ num_heads: input.num_heads,+ num_groups: input.num_kv_heads,+ suffix_length: input.suffix_length,+ sequence_length: input.sequence_length,+ segment_prefix_length,+ max_sequence_length: input.sequence_length,+ ring_params: None,+ head_dim: input.head_dim,+ sliding_window_size: None,+ is_causal: input.do_causal,+ scale: input.scale,+ k_head_stride: (input.sequence_length * input.head_dim) as u64,+ k_seq_stride: input.head_dim as u64,+ v_head_stride: (input.sequence_length * input.head_dim) as u64,+ v_seq_stride: input.head_dim as u64,+ };++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ block.encode(&mut encoder, args).expect("Failed to encode AttentionGemm");+ encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&output_allocation)}fn test_internal<T: ArrayElement + Float + Debug + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rsindex 71ed83fd0..ac8e23339 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_single_pass_test.rs@@ -1,19 +1,22 @@-use std::{- fmt::{Debug, Display},- ops::{Deref, DerefMut},-};+use std::fmt::{Debug, Display};+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{- ArrayContextExt, ArrayElement, DataType,+ ArrayElement, DataType,backends::{- common::{Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},cpu::Cpu,},};-use half::{bf16, f16};-use num_traits::Float;-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+ uzu_test,+};struct Input<T: ArrayElement + Float> {queries: Box<[T]>,@@ -90,19 +93,19 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {).expect("Failed to create AttentionSinglePassKernel");- let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");- let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");- let values_array = context.create_array_from(&[input.values.len()], &input.values, "");+ let queries_allocation = alloc_allocation_with_data::<B, T>(&context, &input.queries);+ let keys_allocation = alloc_allocation_with_data::<B, T>(&context, &input.keys);+ let values_allocation = alloc_allocation_with_data::<B, T>(&context, &input.values);let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;- let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");+ let mut output_allocation = alloc_allocation::<B, T>(&context, output_size);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- queries_array.buffer().borrow().deref(),- keys_array.buffer().borrow().deref(),- values_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ &queries_allocation,+ &keys_allocation,+ &values_allocation,+ &mut output_allocation,input.gqa_factor,input.sequence_length,input.sequence_length * input.head_dim,@@ -111,16 +114,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {input.head_dim,None,input.scale,- None::<&B::DenseBuffer>,+ None::<&Allocation<B>>,None,- None::<&B::DenseBuffer>,+ None::<&Allocation<B>>,input.num_heads,input.suffix_length,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&output_allocation)}fn test_internal<T: ArrayElement + Float + Debug + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rsindex 5416ac401..64102c020 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_two_pass_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -100,27 +99,33 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassI).expect("Failed to create AttentionTwoPass1Kernel");- let queries_array = context.create_array_from(&[input.queries.len()], &input.queries, "");- let keys_array = context.create_array_from(&[input.keys.len()], &input.keys, "");- let values_array = context.create_array_from(&[input.values.len()], &input.values, "");+ let queries_array = context.create_array_from(&[input.queries.len()], &input.queries);+ let keys_array = context.create_array_from(&[input.keys.len()], &input.keys);+ let values_array = context.create_array_from(&[input.values.len()], &input.values);let total_offsets = (input.suffix_length * input.num_heads) as usize;let partials_size = total_offsets * TOTAL_BLOCKS_COUNT as usize * input.head_dim as usize;let sums_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;let maxs_size = total_offsets * TOTAL_BLOCKS_COUNT as usize;- let partials_array = context.create_array_uninitialized(&[partials_size], DataType::F32, "");- let sums_array = context.create_array_uninitialized(&[sums_size], DataType::F32, "");- let maxs_array = context.create_array_uninitialized(&[maxs_size], DataType::F32, "");+ let mut partials = context+ .create_array_uninitialized(&[partials_size], DataType::F32)+ .into_allocation();+ let mut sums = context+ .create_array_uninitialized(&[sums_size], DataType::F32)+ .into_allocation();+ let mut maxs = context+ .create_array_uninitialized(&[maxs_size], DataType::F32)+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- queries_array.buffer().borrow().deref(),- keys_array.buffer().borrow().deref(),- values_array.buffer().borrow().deref(),- partials_array.buffer().borrow_mut().deref_mut(),- sums_array.buffer().borrow_mut().deref_mut(),- maxs_array.buffer().borrow_mut().deref_mut(),+ queries_array.allocation(),+ keys_array.allocation(),+ values_array.allocation(),+ &mut partials,+ &mut sums,+ &mut maxs,input.gqa_factor,input.sequence_length,input.sequence_length * input.head_dim,@@ -131,17 +136,17 @@ fn get_first_pass_output<T: ArrayElement + Float, B: Backend>(input: &FirstPassIinput.scale,input.num_heads,input.suffix_length,- None::<&B::DenseBuffer>,+ None::<&backend_uzu::backends::common::Allocation<B>>,None,- None::<&B::DenseBuffer>,+ None::<&backend_uzu::backends::common::Allocation<B>>,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();FirstPassOutput {- partials: partials_array.as_slice().to_vec(),- sums: sums_array.as_slice().to_vec(),- maxs: maxs_array.as_slice().to_vec(),+ partials: crate::common::helpers::allocation_to_vec(&partials),+ sums: crate::common::helpers::allocation_to_vec(&sums),+ maxs: crate::common::helpers::allocation_to_vec(&maxs),}}@@ -199,26 +204,28 @@ fn get_second_pass_output<T: ArrayElement + Float, B: Backend>(input: &SecondPas<<B as Backend>::Kernels as Kernels>::AttentionTwoPass2Kernel::new(&context, T::data_type(), input.head_dim).expect("Failed to create AttentionTwoPass2Kernel");- let partials_array = context.create_array_from(&[input.partials.len()], &input.partials, "");- let sums_array = context.create_array_from(&[input.sums.len()], &input.sums, "");- let maxs_array = context.create_array_from(&[input.maxs.len()], &input.maxs, "");+ let partials_array = context.create_array_from(&[input.partials.len()], &input.partials);+ let sums_array = context.create_array_from(&[input.sums.len()], &input.sums);+ let maxs_array = context.create_array_from(&[input.maxs.len()], &input.maxs);let output_size = (input.suffix_length * input.num_heads * input.head_dim) as usize;- let output_array = context.create_array_uninitialized(&[output_size], T::data_type(), "");+ let mut output = context+ .create_array_uninitialized(&[output_size], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- partials_array.buffer().borrow().deref(),- sums_array.buffer().borrow().deref(),- maxs_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ partials_array.allocation(),+ sums_array.allocation(),+ maxs_array.allocation(),+ &mut output,input.num_heads,input.suffix_length,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}// --- First pass tests ---diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rsindex b651c57c4..d7aaed133 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -41,21 +40,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let cache_size = input.num_groups as usize * input.max_sequence_length as usize * input.head_dim as usize;- let rotated_keys_array = input.rotated_keys.as_ref().map(|rk| context.create_array_from(&[rk.len()], rk, ""));- let rotated_keys_buf_rc = rotated_keys_array.as_ref().map(|a| a.buffer());- let rotated_keys_buf_borrow = rotated_keys_buf_rc.as_ref().map(|rc| rc.borrow());- let rotated_keys_buffer: Option<&B::DenseBuffer> = rotated_keys_buf_borrow.as_ref().map(|b| b.deref());+ let rotated_keys = input+ .rotated_keys+ .as_ref()+ .map(|rk| context.create_array_from(&[rk.len()], rk).into_allocation());- let qkv_array = context.create_array_from(&[input.qkv.len()], &input.qkv, "");- let key_cache_array = context.create_array_from(&[cache_size], &input.key_cache, "");- let value_cache_array = context.create_array_from(&[cache_size], &input.value_cache, "");+ let qkv_array = context.create_array_from(&[input.qkv.len()], &input.qkv);+ let mut key_cache = context+ .create_array_from(&[cache_size], &input.key_cache)+ .into_allocation();+ let mut value_cache = context+ .create_array_from(&[cache_size], &input.value_cache)+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- rotated_keys_buffer,- qkv_array.buffer().borrow().deref(),- key_cache_array.buffer().borrow_mut().deref_mut(),- value_cache_array.buffer().borrow_mut().deref_mut(),+ rotated_keys.as_ref(),+ qkv_array.allocation(),+ &mut key_cache,+ &mut value_cache,input.num_groups,input.num_heads,input.head_dim,@@ -66,7 +69,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,);encoder.end_encoding().submit().wait_until_completed().unwrap();- (key_cache_array.as_slice().to_vec(), value_cache_array.as_slice().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&key_cache),+ crate::common::helpers::allocation_to_vec(&value_cache),+ )}fn get_test_data_basic<T: ArrayElement + Float>(keys_in_place: bool) -> Input<T> {diff --git a/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rsindex bcd3df4c9..c0c9e50e3 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs@@ -1,4 +1,4 @@-use std::{fmt::Debug, ops::DerefMut};+use std::fmt::Debug;use backend_uzu::{ArrayContextExt, ArrayElement,@@ -28,20 +28,22 @@ fn get_output<T: ArrayElement + Float, B: Backend>(let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidGateKernel::new(&context, T::data_type()).expect("Failed to create SigmoidGateKernel");- let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice(), "gate");- let output_array = context.create_array_from(&[size], &output_data.to_vec().into_boxed_slice(), "output");+ let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice());+ let mut output = context+ .create_array_from(&[size], &output_data.to_vec().into_boxed_slice())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");let total_elements = config.suffix_length * config.num_heads * config.head_dim;kernel.encode(- &*gate_array.buffer().borrow(),- output_array.buffer().borrow_mut().deref_mut(),+ gate_array.allocation(),+ &mut output,total_elements,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn run_test<T: ArrayElement + Float + Debug>(config: &Config) {diff --git a/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rs b/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rsindex 78a863dba..66835a6bc 100644--- a/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/audio/audio_test.rs@@ -1,7 +1,7 @@#![cfg(metal_backend)]use backend_uzu::{- ArrayContextExt, DataType,+ ArrayContextExt, DataType, allocation_to_vec,backends::{common::{Backend, Context, Encoder, Kernels,@@ -28,15 +28,13 @@ use crate::uzu_test;macro_rules! borrow_array_buffer {($name:ident = $array:expr) => {- let $name = $array.buffer();- let $name = $name.borrow();+ let $name = $array.allocation();};}macro_rules! borrow_array_buffer_mut {($name:ident = $array:expr) => {- let $name = $array.buffer();- let mut $name = $name.borrow_mut();+ let $name = &mut $array;};}@@ -81,19 +79,20 @@ fn audio_conv1d_replicate_matches_reference_f32() {let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07 - 1.3).cos() * 0.2).collect();let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.01 - 0.02).collect();- let mut input = context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32);weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_conv1d_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_conv1d_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);- let output = context.create_array_zeros(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");+ let mut output =+ context.create_array_zeros(&[batch_size, cout, seq_len_out], DataType::F32).into_allocation();let expected = conv1d_reference(Conv1dSpec {input: &input_values,weight: &weight_values,@@ -119,11 +118,11 @@ fn audio_conv1d_replicate_matches_reference_f32() {borrow_array_buffer_mut!(output_buffer = output);borrow_array_buffer!(lengths_buffer = lengths);kernel.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input_buffer,+ weight_buffer,+ bias_buffer,+ output_buffer,+ lengths_buffer,cin as i32,cout as i32,seq_len_in as i32,@@ -138,7 +137,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);for index in 0..output_len {let delta = (expected[index] - got[index]).abs();assert!(@@ -174,19 +173,21 @@ fn audio_causal_conv1d_matches_reference_f32() {let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.02 - 0.5).cos() * 0.1).collect();let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.001 - 0.002).collect();- let mut input = context.create_array_zeros(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len], DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32);weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv1d_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths_array = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");+ let mut lengths_array = context.create_array_zeros(&[batch_size], DataType::I32);lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);- let output = context.create_array_zeros(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");+ let mut output = context+ .create_array_zeros(&[batch_size, cout, seq_len], DataType::F32)+ .into_allocation();let expected = causal_conv1d_reference(CausalConv1dSpec {input: &input_values,weight: &weight_values,@@ -208,11 +209,11 @@ fn audio_causal_conv1d_matches_reference_f32() {borrow_array_buffer_mut!(output_buffer = output);borrow_array_buffer!(lengths_buffer = lengths_array);kernel.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input_buffer,+ weight_buffer,+ bias_buffer,+ output_buffer,+ lengths_buffer,cin as i32,cout as i32,seq_len as i32,@@ -224,7 +225,7 @@ fn audio_causal_conv1d_matches_reference_f32() {);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);for index in 0..output_len {let delta = (expected[index] - got[index]).abs();assert!(@@ -264,27 +265,25 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {let bias_values: Vec<f32> = vec![0.01, -0.02];let mut input =- context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");+ context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);let mut weight = context.create_array_zeros(&[cout, cin / groups, kernel_size],- DataType::F32,- "audio_causal_conv_transpose_weight",- );+ DataType::F32);weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);- let output = context.create_array_zeros(+ let mut output = context+ .create_array_zeros(&[batch_size, cout, seq_len_out],- DataType::F32,- "audio_causal_conv_transpose_output",- );+ DataType::F32)+ .into_allocation();let expected = causal_conv_transpose1d_reference(CausalConvTranspose1dSpec {input: &input_values,weight: &weight_values,@@ -307,11 +306,11 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {borrow_array_buffer_mut!(output_buffer = output);borrow_array_buffer!(lengths_buffer = lengths);kernel.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input_buffer,+ weight_buffer,+ bias_buffer,+ output_buffer,+ lengths_buffer,cin as i32,cout as i32,seq_len_in as i32,@@ -323,7 +322,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);for index in 0..output_len {let delta = (expected[index] - got[index]).abs();assert!(@@ -393,28 +392,26 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {("NSC", 1_i32, [batch_size, seq_len_in, cin], input_values_nsc.as_slice()),] {let mut input =- context.create_array_zeros(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");+ context.create_array_zeros(&input_shape, DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(input_values);let mut weight = context.create_array_zeros(&[cout, cin / groups, kernel_size],- DataType::F32,- "audio_causal_conv_transpose_lalamo_weight",- );+ DataType::F32);weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32);bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);let mut lengths =- context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");+ context.create_array_zeros(&[batch_size], DataType::I32);lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);- let output = context.create_array_zeros(+ let mut output = context+ .create_array_zeros(&[batch_size, cout, seq_len_out],- DataType::F32,- "audio_causal_conv_transpose_lalamo_output",- );+ DataType::F32)+ .into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);@@ -423,11 +420,11 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {borrow_array_buffer_mut!(output_buffer = output);borrow_array_buffer!(lengths_buffer = lengths);kernel.encode(- &*input_buffer,- &*weight_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input_buffer,+ weight_buffer,+ bias_buffer,+ output_buffer,+ lengths_buffer,cin as i32,cout as i32,seq_len_in as i32,@@ -441,7 +438,7 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);for index in 0..output_len {let delta = (expected[index] - got[index]).abs();assert!(@@ -465,20 +462,20 @@ fn audio_leaky_relu_matches_reference_f32() {let eps = 1e-6f32;let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();- let mut input = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");+ let mut input = context.create_array_zeros(&[1, 1, n], DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let alpha = context.create_array_zeros(&[1], DataType::F32, "audio_leaky_relu_alpha");- let output = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");+ let alpha = context.create_array_zeros(&[1], DataType::F32);+ let mut output = context.create_array_zeros(&[1, 1, n], DataType::F32).into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);borrow_array_buffer!(alpha_buffer = alpha);borrow_array_buffer_mut!(output_buffer = output);kernel.encode(- &*input_buffer,- &*alpha_buffer,- &mut *output_buffer,+ input_buffer,+ alpha_buffer,+ output_buffer,1,n as i32,0,@@ -489,7 +486,7 @@ fn audio_leaky_relu_matches_reference_f32() {);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);for index in 0..n {let expected = if input_values[index] >= 0.0 {input_values[index]@@ -510,18 +507,18 @@ fn audio_tanh_matches_reference_f32() {let n = 1024usize;let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();- let mut input = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_input");+ let mut input = context.create_array_zeros(&[n], DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let output = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_output");+ let mut output = context.create_array_zeros(&[n], DataType::F32).into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);borrow_array_buffer_mut!(output_buffer = output);- kernel.encode(Some(&*input_buffer), &mut *output_buffer, n as u32, ActivationType::TANH, command_buffer);+ kernel.encode(Some(input_buffer), output_buffer, n as u32, ActivationType::TANH, command_buffer);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);for index in 0..n {let expected = input_values[index].tanh();let delta = (expected - got[index]).abs();@@ -539,22 +536,22 @@ fn audio_add_matches_reference_f32() {let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();- let mut a = context.create_array_zeros(&[n], DataType::F32, "audio_add_a");+ let mut a = context.create_array_zeros(&[n], DataType::F32);a.as_slice_mut::<f32>().copy_from_slice(&a_values);- let mut b = context.create_array_zeros(&[n], DataType::F32, "audio_add_b");+ let mut b = context.create_array_zeros(&[n], DataType::F32);b.as_slice_mut::<f32>().copy_from_slice(&b_values);- let sum = context.create_array_zeros(&[n], DataType::F32, "audio_add_sum");+ let mut sum = context.create_array_zeros(&[n], DataType::F32).into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(a_buffer = a);borrow_array_buffer!(b_buffer = b);borrow_array_buffer_mut!(sum_buffer = sum);- add_kernel.encode(&*a_buffer, &*b_buffer, &mut *sum_buffer, n as i32, command_buffer);+ add_kernel.encode(a_buffer, b_buffer, sum_buffer, n as i32, command_buffer);});- let sum_got = sum.as_slice::<f32>();+ let sum_got = allocation_to_vec::<Metal, f32>(&sum);for index in 0..n {let expected_sum = a_values[index] + b_values[index];@@ -587,22 +584,24 @@ fn audio_half_snake_matches_reference_f32() {let alpha_values: Vec<f32> = (0..snake_channels).map(|i| 0.5 + i as f32 * 0.1).collect();let mut input =- context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");+ context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut alpha = context.create_array_zeros(&[snake_channels], DataType::F32, "audio_half_snake_alpha");+ let mut alpha = context.create_array_zeros(&[snake_channels], DataType::F32);alpha.as_slice_mut::<f32>().copy_from_slice(&alpha_values);- let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");+ let mut output = context+ .create_array_zeros(&[batch_size, channels, seq_len], DataType::F32)+ .into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);borrow_array_buffer!(alpha_buffer = alpha);borrow_array_buffer_mut!(output_buffer = output);kernel.encode(- &*input_buffer,- &*alpha_buffer,- &mut *output_buffer,+ input_buffer,+ alpha_buffer,+ output_buffer,channels as i32,seq_len as i32,snake_channels as i32,@@ -613,7 +612,7 @@ fn audio_half_snake_matches_reference_f32() {);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);let expected = half_snake_reference(HalfSnakeSpec {input: &input_values,alpha: &alpha_values,@@ -692,19 +691,21 @@ fn audio_norm_ncs_matches_reference_f32() {};for subtract_mean in [false, true] {- let mut input = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");+ let mut input = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut scales = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_scales");+ let mut scales = context.create_array_zeros(&[channels], DataType::F32);scales.as_slice_mut::<f32>().copy_from_slice(&scales_values);- let mut bias = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_bias");+ let mut bias = context.create_array_zeros(&[channels], DataType::F32);bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_norm_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);- let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");+ let mut output = context+ .create_array_zeros(&[batch_size, channels, seq_len], DataType::F32)+ .into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);@@ -713,11 +714,11 @@ fn audio_norm_ncs_matches_reference_f32() {borrow_array_buffer_mut!(output_buffer = output);borrow_array_buffer!(lengths_buffer = lengths);kernel.encode(- &*input_buffer,- &*scales_buffer,- &*bias_buffer,- &mut *output_buffer,- &*lengths_buffer,+ input_buffer,+ scales_buffer,+ bias_buffer,+ output_buffer,+ lengths_buffer,channels as i32,seq_len as i32,epsilon,@@ -731,7 +732,7 @@ fn audio_norm_ncs_matches_reference_f32() {);});- let got = output.as_slice::<f32>();+ let got = allocation_to_vec::<Metal, f32>(&output);let expected = reference(subtract_mean);for index in 0..input_len {let delta = (expected[index] - got[index]).abs();@@ -758,29 +759,29 @@ fn audio_fsq_decode_matches_reference() {let num_levels: Vec<i32> = vec![8, 5];let dim_base_index: Vec<i32> = vec![1, 8];- let mut tokens = context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");+ let mut tokens = context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32);tokens.as_slice_mut::<i32>().copy_from_slice(&[0, 7, 11, // g0 tokens at t0..t23, 5, 9, // g1 tokens at t0..t2]);- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);lengths.as_slice_mut::<i32>().copy_from_slice(&[2]);- let output = context.create_array_zeros(+ let mut output = context+ .create_array_zeros(&[batch_size, num_groups * codebook_dim, seq_len],- DataType::F32,- "audio_fsq_output",- );+ DataType::F32)+ .into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(tokens_buffer = tokens);borrow_array_buffer_mut!(output_buffer = output);borrow_array_buffer!(lengths_buffer = lengths);kernel.encode(- &*tokens_buffer,- &mut *output_buffer,- &*lengths_buffer,+ tokens_buffer,+ output_buffer,+ lengths_buffer,num_groups as i32,seq_len as i32,codebook_dim as i32,@@ -794,7 +795,7 @@ fn audio_fsq_decode_matches_reference() {let expected =fsq_decode_reference(&[0, 7, 11, 3, 5, 9], &[2], batch_size, num_groups, seq_len, codebook_dim, &num_levels).expect("reference decode");- let actual = output.as_slice::<f32>();+ let actual = allocation_to_vec::<Metal, f32>(&output);for (index, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() {let delta = (a - e).abs();@@ -826,25 +827,24 @@ fn audio_fsq_encode_matches_reference() {let mut input = context.create_array_zeros(&[batch_size, num_groups * codebook_dim, seq_len],- DataType::F32,- "audio_fsq_encode_input",- );+ DataType::F32);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32);lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);- let tokens =- context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");+ let mut tokens = context+ .create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32)+ .into_allocation();run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);borrow_array_buffer_mut!(tokens_buffer = tokens);borrow_array_buffer!(lengths_buffer = lengths);kernel.encode(- &*input_buffer,- &mut *tokens_buffer,- &*lengths_buffer,+ input_buffer,+ tokens_buffer,+ lengths_buffer,num_groups as i32,seq_len as i32,codebook_dim as i32,@@ -868,7 +868,7 @@ fn audio_fsq_encode_matches_reference() {eps,).expect("reference encode");- let actual = tokens.as_slice::<i32>();+ let actual = allocation_to_vec::<Metal, i32>(&tokens);assert_eq!(actual, expected);}diff --git a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rsindex 88d3007ca..78b32d9e2 100644--- a/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/delta_net_test.rs@@ -1,7 +1,5 @@#![cfg(metal_backend)]-use std::ops::{Deref, DerefMut};-use backend_uzu::{ArrayContextExt, DataType,backends::{@@ -30,20 +28,20 @@ fn run_conv_update<B: Backend>() -> (Vec<f32>, Vec<f32>) {let context = B::Context::new().expect("Failed to create context");- let w_array = context.create_array_from(&[w.len()], w, "w");- let b_array = context.create_array_from(&[b.len()], b, "b");- let in_out_array = context.create_array_from(&[in_proj.len()], in_proj, "in_out");- let state_array = context.create_array_from(&[state.len()], state, "state");+ let w_array = context.create_array_from(&[w.len()], w);+ let b_array = context.create_array_from(&[b.len()], b);+ let mut in_out = context.create_array_from(&[in_proj.len()], in_proj).into_allocation();+ let mut state_allocation = context.create_array_from(&[state.len()], state).into_allocation();let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetConvUpdateKernel::new(&context, DataType::F32, true).expect("Failed to create kernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- w_array.buffer().borrow().deref(),- Some(b_array.buffer().borrow().deref()),- in_out_array.buffer().borrow_mut().deref_mut(),- state_array.buffer().borrow_mut().deref_mut(),+ w_array.allocation(),+ Some(b_array.allocation()),+ &mut in_out,+ &mut state_allocation,kernel_size,conv_dim,state_stride,@@ -51,8 +49,8 @@ fn run_conv_update<B: Backend>();encoder.end_encoding().submit().wait_until_completed().unwrap();- let out = in_out_array.as_slice::<f32>()[..conv_dim as usize].to_vec();- let new_state = state_array.as_slice::<f32>().to_vec();+ let out = crate::common::helpers::allocation_prefix_to_vec::<B, f32>(&in_out, conv_dim as usize);+ let new_state = crate::common::helpers::allocation_to_vec::<B, f32>(&state_allocation);(out, new_state)}@@ -71,24 +69,24 @@ fn run_delta_net_update<B: Backend>() -> (Vec<f32>, Vec<f32>) {let context = B::Context::new().expect("Failed to create context");- let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");- let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");- let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");- let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");- let state_array = context.create_array_from(&[state.len()], state, "state");- let out_array = context.create_array_zeros(&[value_dim as usize], DataType::F32, "out");+ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj);+ let a_log_array = context.create_array_from(&[a_log.len()], a_log);+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias);+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight);+ let mut state_allocation = context.create_array_from(&[state.len()], state).into_allocation();+ let mut out = context.create_array_zeros(&[value_dim as usize], DataType::F32).into_allocation();let kernel = <<B as Backend>::Kernels as Kernels>::DeltaNetUpdateKernel::new(&context, DataType::F32, head_k_dim).expect("Failed to create kernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- in_proj_array.buffer().borrow().deref(),- a_log_array.buffer().borrow().deref(),- dt_bias_array.buffer().borrow().deref(),- norm_weight_array.buffer().borrow().deref(),- state_array.buffer().borrow_mut().deref_mut(),- out_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.allocation(),+ a_log_array.allocation(),+ dt_bias_array.allocation(),+ norm_weight_array.allocation(),+ &mut state_allocation,+ &mut out,num_v_heads,num_k_heads,head_v_dim,@@ -99,8 +97,8 @@ fn run_delta_net_update<B: Backend>();encoder.end_encoding().submit().wait_until_completed().unwrap();- let out = out_array.as_slice::<f32>().to_vec();- let new_state = state_array.as_slice::<f32>().to_vec();+ let out = crate::common::helpers::allocation_to_vec::<B, f32>(&out);+ let new_state = crate::common::helpers::allocation_to_vec::<B, f32>(&state_allocation);(out, new_state)}@@ -176,14 +174,15 @@ fn test_delta_net_conv_scan() {// Test: Conv1dPack + DeltaNetConvScan on Metallet context = <Metal as Backend>::Context::new().expect("context");- let state_array = context.create_array_from(&[init_state.len()], &init_state, "state");- let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");- let w_array = context.create_array_from(&[w.len()], &w, "w");- let b_array = context.create_array_from(&[b.len()], &b, "b");+ let state_array = context.create_array_from(&[init_state.len()], &init_state);+ let mut in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj).into_allocation();+ let w_array = context.create_array_from(&[w.len()], &w);+ let b_array = context.create_array_from(&[b.len()], &b);let padded_len = (tap_count + suffix_len) * total_proj_dim;- let padded_array = context.create_array_zeros(&[padded_len], DataType::F32, "padded");- let state_out_array = context.create_array_zeros(&[conv_dim * tap_count], DataType::F32, "state_out");+ let mut padded_array = context.create_array_zeros(&[padded_len], DataType::F32).into_allocation();+ let mut state_out_array =+ context.create_array_zeros(&[conv_dim * tap_count], DataType::F32).into_allocation();let pack_kernel =<<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(&context, DataType::F32).expect("pack");@@ -193,9 +192,9 @@ fn test_delta_net_conv_scan() {let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");pack_kernel.encode(- state_array.buffer().borrow().deref(),- in_proj_array.buffer().borrow().deref(),- padded_array.buffer().borrow_mut().deref_mut(),+ state_array.allocation(),+ &in_proj_array,+ &mut padded_array,tap_count as u32,total_proj_dim as u32,suffix_len as u32,@@ -203,11 +202,11 @@ fn test_delta_net_conv_scan() {&mut encoder,);scan_kernel.encode(- padded_array.buffer().borrow().deref(),- w_array.buffer().borrow().deref(),- Some(b_array.buffer().borrow().deref()),- in_proj_array.buffer().borrow_mut().deref_mut(),- state_out_array.buffer().borrow_mut().deref_mut(),+ &padded_array,+ w_array.allocation(),+ Some(b_array.allocation()),+ &mut in_proj_array,+ &mut state_out_array,suffix_len as u32,kernel_size as u32,total_proj_dim as u32,@@ -218,13 +217,13 @@ fn test_delta_net_conv_scan() {);encoder.end_encoding().submit().wait_until_completed().unwrap();- let in_proj_result = in_proj_array.as_slice::<f32>();+ let in_proj_result: Vec<f32> = crate::common::helpers::allocation_to_vec(&in_proj_array);let mut scan_outputs = vec![0.0f32; suffix_len * conv_dim];for t in 0..suffix_len {scan_outputs[t * conv_dim..(t + 1) * conv_dim].copy_from_slice(&in_proj_result[t * total_proj_dim..t * total_proj_dim + conv_dim]);}- let scan_state = state_out_array.as_slice::<f32>().to_vec();+ let scan_state: Vec<f32> = crate::common::helpers::allocation_to_vec(&state_out_array);assert_close(&ref_outputs, &scan_outputs, 1e-4, 1e-3, "ConvScan output");assert_close(&ref_state, &scan_state, 1e-5, 1e-4, "ConvScan state");@@ -308,17 +307,21 @@ fn run_prefill_with_norm_gate(let num_dv_groups = (head_v_dim as u32 + 7) / 8;let context = <Metal as Backend>::Context::new().expect("context");- let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj, "in_proj");- let a_log_array = context.create_array_from(&[a_log.len()], a_log, "a_log");- let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias, "dt_bias");- let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight, "norm_weight");- let state_array = context.create_array_from(&[state.len()], state, "state");- let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");- let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");- let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");-- let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");- let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");+ let in_proj_array = context.create_array_from(&[in_proj.len()], in_proj);+ let a_log_array = context.create_array_from(&[a_log.len()], a_log);+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], dt_bias);+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], norm_weight);+ let mut state_array = context.create_array_from(&[state.len()], state).into_allocation();+ let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32).into_allocation();+ let mut q_norm_array =+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();+ let mut k_norm_array =+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();++ let mut beta_array =+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();+ let mut decay_array =+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&context,@@ -337,13 +340,13 @@ fn run_prefill_with_norm_gate(let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");prep_k.encode(- in_proj_array.buffer().borrow().deref(),- a_log_array.buffer().borrow().deref(),- dt_bias_array.buffer().borrow().deref(),- q_norm_array.buffer().borrow_mut().deref_mut(),- k_norm_array.buffer().borrow_mut().deref_mut(),- beta_array.buffer().borrow_mut().deref_mut(),- decay_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.allocation(),+ a_log_array.allocation(),+ dt_bias_array.allocation(),+ &mut q_norm_array,+ &mut k_norm_array,+ &mut beta_array,+ &mut decay_array,num_v_heads as u32,num_k_heads as u32,key_dim as u32,@@ -352,13 +355,13 @@ fn run_prefill_with_norm_gate(&mut encoder,);prefill_k.encode(- q_norm_array.buffer().borrow().deref(),- k_norm_array.buffer().borrow().deref(),- beta_array.buffer().borrow().deref(),- decay_array.buffer().borrow().deref(),- in_proj_array.buffer().borrow().deref(),- state_array.buffer().borrow_mut().deref_mut(),- out_array.buffer().borrow_mut().deref_mut(),+ &q_norm_array,+ &k_norm_array,+ &beta_array,+ &decay_array,+ in_proj_array.allocation(),+ &mut state_array,+ &mut out_array,num_v_heads as u32,num_k_heads as u32,head_v_dim as u32,@@ -369,9 +372,9 @@ fn run_prefill_with_norm_gate(&mut encoder,);norm_k.encode(- out_array.buffer().borrow_mut().deref_mut(),- in_proj_array.buffer().borrow().deref(),- norm_weight_array.buffer().borrow().deref(),+ &mut out_array,+ in_proj_array.allocation(),+ norm_weight_array.allocation(),num_v_heads as u32,head_v_dim as u32,value_dim as u32,@@ -383,7 +386,10 @@ fn run_prefill_with_norm_gate();encoder.end_encoding().submit().wait_until_completed().unwrap();- (out_array.as_slice::<f32>().to_vec(), state_array.as_slice::<f32>().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&out_array),+ crate::common::helpers::allocation_to_vec(&state_array),+ )}fn test_prefill_norm_gate_impl(@@ -469,13 +475,15 @@ fn test_delta_net_prefill_prep() {// CPU reference via Kernels traitlet cpu_ctx = <Cpu as Backend>::Context::new().expect("cpu context");- let cpu_in_proj = cpu_ctx.create_array_from(&[in_proj.len()], &in_proj, "in_proj");- let cpu_a_log = cpu_ctx.create_array_from(&[a_log.len()], &a_log, "a_log");- let cpu_dt_bias = cpu_ctx.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");- let cpu_q = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q");- let cpu_k = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k");- let cpu_beta = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");- let cpu_decay = cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");+ let cpu_in_proj = cpu_ctx.create_array_from(&[in_proj.len()], &in_proj);+ let cpu_a_log = cpu_ctx.create_array_from(&[a_log.len()], &a_log);+ let cpu_dt_bias = cpu_ctx.create_array_from(&[dt_bias.len()], &dt_bias);+ let mut cpu_q = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();+ let mut cpu_k = cpu_ctx.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();+ let mut cpu_beta =+ cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();+ let mut cpu_decay =+ cpu_ctx.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();let cpu_prep = <<Cpu as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&cpu_ctx,@@ -485,13 +493,13 @@ fn test_delta_net_prefill_prep() {.unwrap();let mut cpu_enc = Encoder::new(cpu_ctx.as_ref()).expect("encoder");cpu_prep.encode(- cpu_in_proj.buffer().borrow().deref(),- cpu_a_log.buffer().borrow().deref(),- cpu_dt_bias.buffer().borrow().deref(),- cpu_q.buffer().borrow_mut().deref_mut(),- cpu_k.buffer().borrow_mut().deref_mut(),- cpu_beta.buffer().borrow_mut().deref_mut(),- cpu_decay.buffer().borrow_mut().deref_mut(),+ cpu_in_proj.allocation(),+ cpu_a_log.allocation(),+ cpu_dt_bias.allocation(),+ &mut cpu_q,+ &mut cpu_k,+ &mut cpu_beta,+ &mut cpu_decay,num_v_heads as u32,num_k_heads as u32,key_dim as u32,@@ -501,21 +509,25 @@ fn test_delta_net_prefill_prep() {);cpu_enc.end_encoding().submit().wait_until_completed().unwrap();- let ref_q = cpu_q.as_slice::<f32>().to_vec();- let ref_k = cpu_k.as_slice::<f32>().to_vec();- let ref_beta = cpu_beta.as_slice::<f32>().to_vec();- let ref_decay = cpu_decay.as_slice::<f32>().to_vec();+ let ref_q: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_q);+ let ref_k: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_k);+ let ref_beta: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_beta);+ let ref_decay: Vec<f32> = crate::common::helpers::allocation_to_vec(&cpu_decay);// Metallet context = <Metal as Backend>::Context::new().expect("context");- let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");- let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");- let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");- let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");- let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");-- let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");- let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj);+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log);+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias);+ let mut q_norm_array =+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();+ let mut k_norm_array =+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();++ let mut beta_array =+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();+ let mut decay_array =+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();let prep_k = <<Metal as Backend>::Kernels as Kernels>::DeltaNetPrefillPrepKernel::new(&context,@@ -526,13 +538,13 @@ fn test_delta_net_prefill_prep() {let mut encoder = Encoder::new(context.as_ref()).expect("encoder");prep_k.encode(- in_proj_array.buffer().borrow().deref(),- a_log_array.buffer().borrow().deref(),- dt_bias_array.buffer().borrow().deref(),- q_norm_array.buffer().borrow_mut().deref_mut(),- k_norm_array.buffer().borrow_mut().deref_mut(),- beta_array.buffer().borrow_mut().deref_mut(),- decay_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.allocation(),+ a_log_array.allocation(),+ dt_bias_array.allocation(),+ &mut q_norm_array,+ &mut k_norm_array,+ &mut beta_array,+ &mut decay_array,num_v_heads as u32,num_k_heads as u32,key_dim as u32,@@ -542,15 +554,15 @@ fn test_delta_net_prefill_prep() {);encoder.end_encoding().submit().wait_until_completed().unwrap();- let gpu_q = q_norm_array.as_slice::<f32>();- let gpu_k = k_norm_array.as_slice::<f32>();- let gpu_beta = beta_array.as_slice::<f32>();- let gpu_decay = decay_array.as_slice::<f32>();+ let gpu_q: Vec<f32> = crate::common::helpers::allocation_to_vec(&q_norm_array);+ let gpu_k: Vec<f32> = crate::common::helpers::allocation_to_vec(&k_norm_array);+ let gpu_beta: Vec<f32> = crate::common::helpers::allocation_to_vec(&beta_array);+ let gpu_decay: Vec<f32> = crate::common::helpers::allocation_to_vec(&decay_array);- assert_close(gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");- assert_close(gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");- assert_close(gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");- assert_close(gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");+ assert_close(&gpu_q, &ref_q, 1e-4, 1e-3, "prep q_norm");+ assert_close(&gpu_k, &ref_k, 1e-4, 1e-3, "prep k_norm");+ assert_close(&gpu_beta, &ref_beta, 1e-4, 1e-3, "prep beta");+ assert_close(&gpu_decay, &ref_decay, 1e-4, 1e-3, "prep decay");}#[uzu_test]@@ -576,16 +588,20 @@ fn bench_delta_net_prefill() {let norm_weight: Vec<f32> = (0..head_v_dim).map(|i| 0.9 + (i as f32) * 0.001).collect();let context = <Metal as Backend>::Context::new().expect("context");- let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj, "in_proj");- let a_log_array = context.create_array_from(&[a_log.len()], &a_log, "a_log");- let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias, "dt_bias");- let norm_weight_array = context.create_array_from(&[norm_weight.len()], &norm_weight, "norm_weight");- let out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32, "out");- let q_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "q_norm");- let k_norm_array = context.create_array_zeros(&[suffix_len * key_dim], DataType::F32, "k_norm");-- let beta_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "beta");- let decay_array = context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32, "decay");+ let in_proj_array = context.create_array_from(&[in_proj.len()], &in_proj);+ let a_log_array = context.create_array_from(&[a_log.len()], &a_log);+ let dt_bias_array = context.create_array_from(&[dt_bias.len()], &dt_bias);+ let norm_weight_array = context.create_array_from(&[norm_weight.len()], &norm_weight);+ let mut out_array = context.create_array_zeros(&[suffix_len * value_dim], DataType::F32).into_allocation();+ let mut q_norm_array =+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();+ let mut k_norm_array =+ context.create_array_zeros(&[suffix_len * key_dim], DataType::F32).into_allocation();++ let mut beta_array =+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();+ let mut decay_array =+ context.create_array_zeros(&[suffix_len * num_v_heads], DataType::F32).into_allocation();let num_dv_groups = (head_v_dim as u32 + 7) / 8;@@ -613,13 +629,13 @@ fn bench_delta_net_prefill() {let prep_result = run_perf_with_warmup("prep_only", 5, 50, || {let mut encoder = Encoder::new(context.as_ref()).expect("encoder");prep_k.encode(- in_proj_array.buffer().borrow().deref(),- a_log_array.buffer().borrow().deref(),- dt_bias_array.buffer().borrow().deref(),- q_norm_array.buffer().borrow_mut().deref_mut(),- k_norm_array.buffer().borrow_mut().deref_mut(),- beta_array.buffer().borrow_mut().deref_mut(),- decay_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.allocation(),+ a_log_array.allocation(),+ dt_bias_array.allocation(),+ &mut q_norm_array,+ &mut k_norm_array,+ &mut beta_array,+ &mut decay_array,num_v_heads as u32,num_k_heads as u32,key_dim as u32,@@ -632,20 +648,19 @@ fn bench_delta_net_prefill() {prep_result.print();// Benchmark prep + prefill + norm_gate (production path)- let mut state_array = context.create_array_zeros(&[state_size], DataType::F32, "state");+ let mut state_array = context.create_array_zeros(&[state_size], DataType::F32).into_allocation();let prefill_result = run_perf_with_warmup("prep+prefill+norm_gate", 5, 50, || {- state_array.as_slice_mut::<f32>().fill(0.0);-let mut encoder = Encoder::new(context.as_ref()).expect("encoder");+ encoder.encode_fill(&mut state_array, 0);prep_k.encode(- in_proj_array.buffer().borrow().deref(),- a_log_array.buffer().borrow().deref(),- dt_bias_array.buffer().borrow().deref(),- q_norm_array.buffer().borrow_mut().deref_mut(),- k_norm_array.buffer().borrow_mut().deref_mut(),- beta_array.buffer().borrow_mut().deref_mut(),- decay_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.allocation(),+ a_log_array.allocation(),+ dt_bias_array.allocation(),+ &mut q_norm_array,+ &mut k_norm_array,+ &mut beta_array,+ &mut decay_array,num_v_heads as u32,num_k_heads as u32,key_dim as u32,@@ -654,13 +669,13 @@ fn bench_delta_net_prefill() {&mut encoder,);prefill_k.encode(- q_norm_array.buffer().borrow().deref(),- k_norm_array.buffer().borrow().deref(),- beta_array.buffer().borrow().deref(),- decay_array.buffer().borrow().deref(),- in_proj_array.buffer().borrow().deref(),- state_array.buffer().borrow_mut().deref_mut(),- out_array.buffer().borrow_mut().deref_mut(),+ &q_norm_array,+ &k_norm_array,+ &beta_array,+ &decay_array,+ in_proj_array.allocation(),+ &mut state_array,+ &mut out_array,num_v_heads as u32,num_k_heads as u32,head_v_dim as u32,@@ -671,9 +686,9 @@ fn bench_delta_net_prefill() {&mut encoder,);norm_k.encode(- out_array.buffer().borrow_mut().deref_mut(),- in_proj_array.buffer().borrow().deref(),- norm_weight_array.buffer().borrow().deref(),+ &mut out_array,+ in_proj_array.allocation(),+ norm_weight_array.allocation(),num_v_heads as u32,head_v_dim as u32,value_dim as u32,diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rsindex 8d67f4c70..89342d224 100644--- a/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/embedding/embedding_rows_sum_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -105,16 +104,18 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let kernel = <<B as Backend>::Kernels as Kernels>::EmbeddingRowsSumKernel::new(&context, T::data_type()).expect("Failed to create EmbeddingRowsSumKernel");- let token_indices_array = context.create_array_from(&[input.num_rows as usize], &input.token_indices, "");+ let token_indices_array = context.create_array_from(&[input.num_rows as usize], &input.token_indices);let weights_array =- context.create_array_from(&[input.total_rows as usize, input.model_dim as usize], &input.weights, "");- let output_array = context.create_array_uninitialized(&[input.model_dim as usize], T::data_type(), "");+ context.create_array_from(&[input.total_rows as usize, input.model_dim as usize], &input.weights);+ let mut output = context+ .create_array_uninitialized(&[input.model_dim as usize], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- token_indices_array.buffer().borrow().deref(),- weights_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ token_indices_array.allocation(),+ weights_array.allocation(),+ &mut output,input.num_rows,input.total_rows,input.model_dim,@@ -123,7 +124,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_basic<T: ArrayElement + Float + Debug + Display>() {diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rsindex 0ac84188a..23012072c 100644--- a/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/embedding/full_precision_embedding_test.rs@@ -1,6 +1,5 @@use std::{fmt::Debug,- ops::{Deref, DerefMut},};use backend_uzu::{@@ -57,15 +56,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {<<B as Backend>::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(&context, T::data_type()).expect("Failed to create FullPrecisionEmbeddingLookupKernel");- let token_ids_array = context.create_array_from(&[input.batch_size], &input.token_ids, "");- let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights, "");- let output_array = context.create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type(), "");+ let token_ids_array = context.create_array_from(&[input.batch_size], &input.token_ids);+ let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights);+ let mut output = context+ .create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");kernel.encode(- token_ids_array.buffer().borrow().deref(),- weights_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ token_ids_array.allocation(),+ weights_array.allocation(),+ &mut output,input.batch_size as u32,input.vocab_size as u32,input.model_dim as u32,@@ -74,7 +75,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test<T: ArrayElement + Float + Debug>() {diff --git a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rsindex ffeaf4649..56ccc1434 100644--- a/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/embedding/quant_embedding_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -120,20 +119,21 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let weights_stride = input.model_dim as usize / packing_divisor;let num_groups = (input.model_dim as usize + input.group_size as usize - 1) / input.group_size as usize;- let token_ids_array = context.create_array_from(&[input.batch_size as usize], &input.token_ids, "");- let weights_array = context.create_array_from(&[input.vocab_size as usize, weights_stride], &input.weights, "");- let scales_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.scales, "");- let biases_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.biases, "");- let output_array =- context.create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type(), "");+ let token_ids_array = context.create_array_from(&[input.batch_size as usize], &input.token_ids);+ let weights_array = context.create_array_from(&[input.vocab_size as usize, weights_stride], &input.weights);+ let scales_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.scales);+ let biases_array = context.create_array_from(&[input.vocab_size as usize, num_groups], &input.biases);+ let mut output = context+ .create_array_uninitialized(&[input.batch_size as usize, input.model_dim as usize], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- token_ids_array.buffer().borrow().deref(),- weights_array.buffer().borrow().deref(),- scales_array.buffer().borrow().deref(),- biases_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ token_ids_array.allocation(),+ weights_array.allocation(),+ scales_array.allocation(),+ biases_array.allocation(),+ &mut output,input.batch_size,input.vocab_size,input.model_dim,@@ -142,7 +142,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_quant_mode<T: ArrayElement + Float + Debug + Display>(quant_mode: QuantizationMode) {diff --git a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rsindex a37515b97..c67f7353b 100644--- a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs@@ -1,6 +1,5 @@use std::{fmt::Debug,- ops::{Deref, DerefMut},};use backend_uzu::{@@ -105,20 +104,22 @@ fn run_kernel<T: ArrayElement + Float, B: Backend>(input: &TestInput<T>) -> Vec<.expect("Failed to create HadamardTransformKernel");let total_elements = input.batch_count * input.channel_count;- let data_array = context.create_array_from(&[total_elements], &input.data, "data");- let factors_array = context.create_array_from::<i32>(&[input.channel_count], &input.factors, "factors");+ let mut data = context+ .create_array_from(&[total_elements], &input.data)+ .into_allocation();+ let factors_array = context.create_array_from::<i32>(&[input.channel_count], &input.factors);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- data_array.buffer().borrow_mut().deref_mut(),- factors_array.buffer().borrow().deref(),+ &mut data,+ factors_array.allocation(),input.channel_count as u32,input.batch_count as u32,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- data_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&data)}fn test_hadamard_transform<T: ArrayElement + Float + Debug>(tolerance: f64) {diff --git a/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rs b/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rsindex bb8a23ad9..f0c644f01 100644--- a/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/kv_cache_update_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::DerefMut,};use backend_uzu::{@@ -45,13 +44,13 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,.expect("Failed to create KVCacheUpdateKernel");let total = input.num_heads as usize * input.max_sequence_length as usize * input.head_dim as usize;- let keys_array = context.create_array_from(&[total], &input.keys, "");- let values_array = context.create_array_from(&[total], &input.values, "");+ let mut keys = context.create_array_from(&[total], &input.keys).into_allocation();+ let mut values = context.create_array_from(&[total], &input.values).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");kernel.encode(- keys_array.buffer().borrow_mut().deref_mut(),- values_array.buffer().borrow_mut().deref_mut(),+ &mut keys,+ &mut values,&input.swaps,input.swaps.len() as u32,input.num_heads,@@ -60,7 +59,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,);encoder.end_encoding().submit().wait_until_completed().unwrap();- (keys_array.as_slice().to_vec(), values_array.as_slice().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&keys),+ crate::common::helpers::allocation_to_vec(&values),+ )}/// Single swap between two different positions.diff --git a/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rsindex a6675dca2..d19549361 100644--- a/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/layer_norm_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -128,23 +127,21 @@ fn get_output<.expect("Failed to create LayerNormKernel");let input_size = input.input.len();- let input_array = context.create_array_from(&[input_size], &input.input, "");- let input_array_buffer_rc = input_array.buffer();- let input_array_borrow = input_array_buffer_rc.borrow();- let input_array_deref = input_array_borrow.deref();- let input_buffer = (!input.in_place).then(|| input_array_deref);-- let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");- let output_array = match input.in_place {- true => context.create_array_from(&[input_size], &input.output, ""),- false => context.create_array_uninitialized(&[input_size], OUT::data_type(), ""),+ let input_buffer = (!input.in_place).then(|| context.create_array_from(&[input_size], &input.input).into_allocation());++ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales);+ let mut output = match input.in_place {+ true => context.create_array_from(&[input_size], &input.output).into_allocation(),+ false => context+ .create_array_uninitialized(&[input_size], OUT::data_type())+ .into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- input_buffer,- scales_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ input_buffer.as_ref(),+ scales_array.allocation(),+ &mut output,input.batch_size,input.model_dim,input.epsilon,@@ -154,7 +151,7 @@ fn get_output<);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_internal<diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rsindex 19ec918c3..018aa3458 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs@@ -1,15 +1,16 @@#![cfg(metal_backend)]use std::{- ops::{Deref, DerefMut},rc::Rc,};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::bf16;use backend_uzu::{ArrayContextExt, ArrayElement,backends::{common::{- Backend, Context, Encoder,+ AllocationType, Backend, Context, Encoder,kernel::{ManualKernels,matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},@@ -19,11 +20,13 @@ use backend_uzu::{metal::{DeviceExt, MatmulDispatchPath, Metal, MetalContext},},};-use criterion::{BenchmarkId, Criterion, Throughput};-use half::bf16;use crate::{- common::{assert::assert_eq_float, type_short_name},+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation_with_data, allocation_to_vec},+ type_short_name,+ },uzu_bench, uzu_test,};@@ -66,21 +69,19 @@ fn get_test_data(fn get_cpu_output(input: &Input) -> Vec<bf16> {let cpu_context = <Cpu as Backend>::Context::new().expect("CPU context");- let left_array = cpu_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");- let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");- let destination_array = if let Some(ref prefill) = input.destination_prefill {- cpu_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")+ let right_array = cpu_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);+ let left_allocation = alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, &input.left);+ let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {+ alloc_allocation_with_data::<Cpu, bf16>(&cpu_context, prefill)} else {- cpu_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")+ cpu_context+ .create_allocation(+ input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),+ AllocationType::Global,+ )+ .expect("Failed to create allocation")};- let left_buffer = left_array.buffer();- let left_ref = left_buffer.borrow();- let right_buffer = right_array.buffer();- let right_ref = right_buffer.borrow();- let destination_buffer = destination_array.buffer();- let mut destination_ref = destination_buffer.borrow_mut();-let argument_c = if input.destination_prefill.is_some() {MatmulArgumentC::Accumulate} else {@@ -93,14 +94,13 @@ fn get_cpu_output(input: &Input) -> Vec<bf16> {let mut encoder = Encoder::new(cpu_context.as_ref()).expect("encoder");matmul_kernel.encode(- &cpu_context,MatmulArguments {- a: left_ref.deref(),+ a: &left_allocation,a_offset: 0,- b: right_ref.deref(),+ b: right_array.allocation(),ab_scale: input.ab_scale,c: argument_c,- d: destination_ref.deref_mut(),+ d: &mut destination_allocation,batch_dim: input.batch_dim as u32,input_dim: input.input_dim as u32,output_dim: input.output_dim as u32,@@ -108,9 +108,7 @@ fn get_cpu_output(input: &Input) -> Vec<bf16> {&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();-- drop(destination_ref);- destination_array.as_slice().to_vec()+ allocation_to_vec(&destination_allocation)}fn get_mpp_output(@@ -118,21 +116,19 @@ fn get_mpp_output(matmul_kernel: &mut <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel,input: &Input,) -> Vec<bf16> {- let left_array = metal_context.create_array_from(&[input.batch_dim, input.input_dim], &input.left, "");- let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right, "");- let destination_array = if let Some(ref prefill) = input.destination_prefill {- metal_context.create_array_from(&[input.batch_dim, input.output_dim], prefill, "")+ let right_array = metal_context.create_array_from(&[input.output_dim, input.input_dim], &input.right);+ let left_allocation = alloc_allocation_with_data::<Metal, bf16>(metal_context, &input.left);+ let mut destination_allocation = if let Some(ref prefill) = input.destination_prefill {+ alloc_allocation_with_data::<Metal, bf16>(metal_context, prefill)} else {- metal_context.create_array_uninitialized(&[input.batch_dim, input.output_dim], bf16::data_type(), "")+ metal_context+ .create_allocation(+ input.batch_dim * input.output_dim * std::mem::size_of::<bf16>(),+ AllocationType::Global,+ )+ .expect("Failed to create allocation")};- let left_buffer = left_array.buffer();- let left_ref = left_buffer.borrow();- let right_buffer = right_array.buffer();- let right_ref = right_buffer.borrow();- let destination_buffer = destination_array.buffer();- let mut destination_ref = destination_buffer.borrow_mut();-let argument_c = if input.destination_prefill.is_some() {MatmulArgumentC::Accumulate} else {@@ -141,14 +137,13 @@ fn get_mpp_output(let mut encoder = Encoder::new(metal_context).expect("encoder");matmul_kernel.encode_with_path(- metal_context,MatmulArguments {- a: left_ref.deref(),+ a: &left_allocation,a_offset: 0,- b: right_ref.deref(),+ b: right_array.allocation(),ab_scale: input.ab_scale,c: argument_c,- d: destination_ref.deref_mut(),+ d: &mut destination_allocation,batch_dim: input.batch_dim as u32,input_dim: input.input_dim as u32,output_dim: input.output_dim as u32,@@ -157,9 +152,7 @@ fn get_mpp_output(MatmulDispatchPath::GemmMpp,);encoder.end_encoding().submit().wait_until_completed().unwrap();-- drop(destination_ref);- destination_array.as_slice().to_vec()+ allocation_to_vec(&destination_allocation)}fn create_metal_context_and_matmul_kernel()@@ -291,10 +284,13 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()));for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {- let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");- let destination_array =- metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());+ let left_allocation = metal_context+ .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("Failed to create allocation");+ let mut destination_allocation = metal_context+ .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("Failed to create allocation");let floating_point_operations = 2 * batch_dim * input_dim * output_dim;benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));@@ -302,26 +298,18 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {benchmark_group.bench_function(BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),|bencher| {- let left_buffer = left_array.buffer();- let left_ref = left_buffer.borrow();- let right_buffer = right_array.buffer();- let right_ref = right_buffer.borrow();- let destination_buffer = destination_array.buffer();-bencher.iter_custom(|iteration_count| {let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();for _ in 0..iteration_count {- let mut destination_ref = destination_buffer.borrow_mut();matmul_kernel.encode_with_path(- &metal_context,MatmulArguments {- a: left_ref.deref(),+ a: &left_allocation,a_offset: 0,- b: right_ref.deref(),+ b: right_array.allocation(),ab_scale: 1.0,c: MatmulArgumentC::None,- d: destination_ref.deref_mut(),+ d: &mut destination_allocation,batch_dim: batch_dim as u32,input_dim: input_dim as u32,output_dim: output_dim as u32,diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rsindex 2f197a59f..e1fa1aa86 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs@@ -1,13 +1,12 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{ArrayContextExt, ArrayElement,backends::{common::{- Backend, Context, Encoder,+ AllocationType, Backend, Context, Encoder,kernel::{ManualKernels,matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},@@ -25,7 +24,13 @@ use num_traits::Float;#[cfg(metal_backend)]use crate::{common::type_short_name, uzu_bench};-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation_with_data, allocation_to_vec},+ },+ uzu_test,+};struct Input<T: ArrayElement + Float> {a: Box<[T]>,@@ -55,40 +60,31 @@ fn get_test_data<T: ArrayElement + Float>((input, expected)}-fn get_output<T: ArrayElement + Float, B: Backend>(- input: &Input<T>,- ab_scale: f32,-) -> Vec<T> {+fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>, ab_scale: f32) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");let m = input.m as u32;let k = input.k as u32;let n = input.n as u32;- let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");- let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");- let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");-- let a_buf = a_array.buffer();- let a_ref = a_buf.borrow();- let b_buf = b_array.buffer();- let b_ref = b_buf.borrow();- let d_buf = d_array.buffer();- let mut d_ref = d_buf.borrow_mut();+ let b_array = context.create_array_from(&[input.n, input.k], &input.b);+ let a_allocation = alloc_allocation_with_data::<B, T>(&context, &input.a);+ let mut d_allocation = context+ .create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global)+ .expect("Failed to create allocation");let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type()).expect("Failed to create MatmulKernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- &context,MatmulArguments {- a: a_ref.deref(),+ a: &a_allocation,a_offset: 0,- b: b_ref.deref(),+ b: b_array.allocation(),ab_scale,c: MatmulArgumentC::None,- d: d_ref.deref_mut(),+ d: &mut d_allocation,batch_dim: m,input_dim: k,output_dim: n,@@ -96,9 +92,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();-- drop(d_ref);- d_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&d_allocation)}fn test<T: ArrayElement + Float + Debug + Display>(@@ -125,12 +119,7 @@ fn test_with_scale<T: ArrayElement + Float + Debug + Display>(let expected = get_output::<T, Cpu>(&input, ab_scale);for_each_non_cpu_backend!(|B| {let output = get_output::<T, B>(&input, ab_scale);- assert_eq_float(- &expected,- &output,- eps,- &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()),- );+ assert_eq_float(&expected, &output, eps, &format!("backend {} ab_scale={ab_scale}", std::any::type_name::<B>()));});}@@ -210,10 +199,13 @@ fn bench_gemm(criterion: &mut Criterion) {criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {- let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");- let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");- let destination_array =- metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");+ let left_allocation = metal_context+ .create_allocation(batch_dim * input_dim * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("Failed to create allocation");+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type());+ let mut destination_allocation = metal_context+ .create_allocation(batch_dim * output_dim * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("Failed to create allocation");let floating_point_operations = 2 * batch_dim * input_dim * output_dim;benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));@@ -221,26 +213,18 @@ fn bench_gemm(criterion: &mut Criterion) {benchmark_group.bench_function(BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),|bencher| {- let left_buffer = left_array.buffer();- let left_ref = left_buffer.borrow();- let right_buffer = right_array.buffer();- let right_ref = right_buffer.borrow();- let destination_buffer = destination_array.buffer();-bencher.iter_custom(|iteration_count| {let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();for _ in 0..iteration_count {- let mut destination_ref = destination_buffer.borrow_mut();matmul_kernel.encode_with_path(- &metal_context,MatmulArguments {- a: left_ref.deref(),+ a: &left_allocation,a_offset: 0,- b: right_ref.deref(),+ b: right_array.allocation(),ab_scale: 1.0,c: MatmulArgumentC::None,- d: destination_ref.deref_mut(),+ d: &mut destination_allocation,batch_dim: batch_dim as u32,input_dim: input_dim as u32,output_dim: output_dim as u32,diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rsindex 8ffaa9b97..351338060 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -1,13 +1,14 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{ArrayContextExt, ArrayElement,backends::{common::{- Backend, Context, Encoder,+ AllocationType, Backend, Context, Encoder,kernel::{ManualKernels,matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},@@ -16,10 +17,11 @@ use backend_uzu::{cpu::Cpu,},};-use half::{bf16, f16};-use num_traits::Float;-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{assert::assert_eq_float, helpers::{alloc_allocation_with_data, allocation_to_vec}},+ uzu_test,+};struct Input<T: ArrayElement + Float> {a: Box<[T]>,@@ -56,30 +58,24 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let k = input.k as u32;let n = input.n as u32;- let a_array = context.create_array_from(&[input.m, input.k], &input.a, "");- let b_array = context.create_array_from(&[input.n, input.k], &input.b, "");- let d_array = context.create_array_uninitialized(&[input.m, input.n], T::data_type(), "");-- let a_buf = a_array.buffer();- let a_ref = a_buf.borrow();- let b_buf = b_array.buffer();- let b_ref = b_buf.borrow();- let d_buf = d_array.buffer();- let mut d_ref = d_buf.borrow_mut();+ let b_array = context.create_array_from(&[input.n, input.k], &input.b);+ let a_allocation = alloc_allocation_with_data::<B, T>(&context, &input.a);+ let mut d_allocation = context+ .create_allocation(input.m * input.n * std::mem::size_of::<T>(), AllocationType::Global)+ .expect("Failed to create allocation");let mut kernel = <B::Kernels as ManualKernels>::MatmulKernel::new(&context, T::data_type()).expect("Failed to create MatmulKernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- &context,MatmulArguments {- a: a_ref.deref(),+ a: &a_allocation,a_offset: 0,- b: b_ref.deref(),+ b: b_array.allocation(),ab_scale: 1.0,c: MatmulArgumentC::None,- d: d_ref.deref_mut(),+ d: &mut d_allocation,batch_dim: m,input_dim: k,output_dim: n,@@ -87,9 +83,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();-- drop(d_ref);- d_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&d_allocation)}fn test<T: ArrayElement + Float + Debug + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rsindex f650b2f70..3413c8151 100644--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_counts_offsets_fused_test.rs@@ -1,5 +1,4 @@-use std::ops::{Deref, DerefMut};-+use rand::{RngExt, SeedableRng, rngs::StdRng};use backend_uzu::{ArrayContextExt, ArrayElement,backends::{@@ -7,7 +6,6 @@ use backend_uzu::{cpu::Cpu,},};-use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::uzu_test;@@ -40,21 +38,22 @@ fn get_output<B: Backend>(let topk_ids_len = (t * k).max(1);let topk_ids_array = if topk_ids.is_empty() {- context.create_array_uninitialized(&[topk_ids_len], i32::data_type(), "topk_ids")+ context.create_array_uninitialized(&[topk_ids_len], i32::data_type())} else {- context.create_array_from(&[topk_ids_len], topk_ids, "topk_ids")+ context.create_array_from(&[topk_ids_len], topk_ids)};- let offsets_array = context.create_array_uninitialized(&[e + 1], u32::data_type(), "offsets");- let sum_k_array = context.create_array_uninitialized(&[1], u32::data_type(), "sum_k");+ let mut offsets = context.create_array_uninitialized(&[e + 1], u32::data_type()).into_allocation();+ let mut sum_k = context.create_array_uninitialized(&[1], u32::data_type()).into_allocation();let num_tiles = e.div_ceil(512).max(1);- let partials_array = context.create_array_uninitialized(&[num_tiles * 512], u32::data_type(), "partials");+ let mut partials =+ context.create_array_uninitialized(&[num_tiles * 512], u32::data_type()).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- topk_ids_array.buffer().borrow().deref(),- offsets_array.buffer().borrow_mut().deref_mut(),- sum_k_array.buffer().borrow_mut().deref_mut(),- partials_array.buffer().borrow_mut().deref_mut(),+ topk_ids_array.allocation(),+ &mut offsets,+ &mut sum_k,+ &mut partials,t as u32,e as u32,k as u32,@@ -62,9 +61,9 @@ fn get_output<B: Backend>();encoder.end_encoding().submit().wait_until_completed().unwrap();- let offsets = offsets_array.as_slice::<u32>().to_vec();- let sum_k = sum_k_array.as_slice::<u32>()[0];- let partials = partials_array.as_slice::<u32>()[..e].to_vec();+ let offsets = crate::common::helpers::allocation_to_vec::<B, u32>(&offsets);+ let sum_k = crate::common::helpers::allocation_to_vec::<B, u32>(&sum_k)[0];+ let partials = crate::common::helpers::allocation_prefix_to_vec::<B, u32>(&partials, e);(offsets, sum_k, partials)}diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rsindex 60fc1b460..e7afacf71 100644--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_finalize_test.rs@@ -1,5 +1,6 @@-use std::ops::{Deref, DerefMut};-+use half::bf16;+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::StdRng};use backend_uzu::{ArrayContextExt, ArrayElement, DataType,backends::{@@ -7,9 +8,6 @@ use backend_uzu::{cpu::Cpu,},};-use half::bf16;-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::{common::{assert::assert_eq_float, helpers::create_context},@@ -27,18 +25,18 @@ struct Input<T: ArrayElement + Float> {fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = create_context::<B>();- let tok2row_array = context.create_array_from(&[input.tok2row.len()], &input.tok2row, "");- let probs_array = context.create_array_from(&[input.probs.len()], &input.probs, "");- let y_partial_array = context.create_array_from(&[input.y_partial.len()], &input.y_partial, "");- let y_out_array = context.create_array_uninitialized(&[input.t * input.d_model], T::data_type(), "");+ let tok2row_array = context.create_array_from(&[input.tok2row.len()], &input.tok2row);+ let probs_array = context.create_array_from(&[input.probs.len()], &input.probs);+ let y_partial_array = context.create_array_from(&[input.y_partial.len()], &input.y_partial);+ let mut y_out = context.create_array_uninitialized(&[input.t * input.d_model], T::data_type()).into_allocation();let finalize = <B::Kernels as Kernels>::MoeFinalizeKernel::new(&context, DataType::BF16).expect("finalize kernel");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");finalize.encode(- tok2row_array.buffer().borrow().deref(),- probs_array.buffer().borrow().deref(),- y_partial_array.buffer().borrow().deref(),- y_out_array.buffer().borrow_mut().deref_mut(),+ tok2row_array.allocation(),+ probs_array.allocation(),+ y_partial_array.allocation(),+ &mut y_out,input.t as u32,input.d_model as u32,input.k as u32,@@ -46,7 +44,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- y_out_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&y_out)}fn test_finalize_internal(diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rsindex 7a7572e16..493348dc4 100644--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_gather_test.rs@@ -1,7 +1,8 @@-use std::ops::{Deref, DerefMut};-+use half::bf16;+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::StdRng};use backend_uzu::{- ArrayContextExt, ArrayElement,+ ArrayElement,backends::{common::{Backend, Encoder,@@ -10,12 +11,12 @@ use backend_uzu::{cpu::Cpu,},};-use half::bf16;-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::{- common::{assert::assert_eq_float, helpers::create_context},+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},+ },uzu_test,};@@ -30,11 +31,11 @@ struct Input<T: ArrayElement + Float> {fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = create_context::<B>();- let x_array = context.create_array_from(&[input.x.len()], &input.x, "");- let ids_array = context.create_array_from(&[input.bucket_ids.len()], &input.bucket_ids, "");let sumk_data: [u32; 1] = [input.sum_k as u32];- let sumk_array = context.create_array_from(&[1], &sumk_data, "");- let x_perm_array = context.create_array_uninitialized(&[input.sum_k * input.d_model], T::data_type(), "");+ let x_allocation = alloc_allocation_with_data::<B, T>(&context, &input.x);+ let ids_allocation = alloc_allocation_with_data::<B, i32>(&context, &input.bucket_ids);+ let sumk_allocation = alloc_allocation_with_data::<B, u32>(&context, &sumk_data);+ let mut x_perm_allocation = alloc_allocation::<B, T>(&context, input.sum_k * input.d_model);let gather = MoeGatherKernels::<B>::new(&context).expect("MoeGatherKernel::new");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -42,10 +43,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {&mut encoder,T::data_type(),MoeGatherArguments {- x_buffer: &x_array.buffer().borrow().deref(),- bucketed_ids_buffer: &ids_array.buffer().borrow().deref(),- x_perm_buffer: x_perm_array.buffer().borrow_mut().deref_mut(),- sumk_buffer: &sumk_array.buffer().borrow().deref(),+ x: &x_allocation,+ bucketed_ids: &ids_allocation,+ x_perm: &mut x_perm_allocation,+ sumk: &sumk_allocation,t: input.t,k: input.sum_k / input.t, // Decompose sum_k into k per tokend_model: input.d_model,@@ -53,7 +54,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- x_perm_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&x_perm_allocation)}fn test_gather_internal(diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rsindex 607256bbf..669f465ad 100644--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_router_topk_test.rs@@ -1,6 +1,5 @@use std::{fmt::Debug,- ops::{Deref, DerefMut},};use backend_uzu::{@@ -28,20 +27,20 @@ fn get_output<B: Backend, T: ArrayElement + Float>() -> (Vec<i32>, Vec<T>) {let ctx = create_context::<B>();- let input_array = ctx.create_array_from(&[input.len()], input, "");- let weights_array = ctx.create_array_from(&[weights.len()], weights, "");- let bias_array = ctx.create_array_from(&[bias.len()], bias, "");- let ids_array = ctx.create_array_uninitialized(&[t * k], DataType::I32, "");- let probs_array = ctx.create_array_uninitialized(&[t * k], T::data_type(), "");+ let input_array = ctx.create_array_from(&[input.len()], input);+ let weights_array = ctx.create_array_from(&[weights.len()], weights);+ let bias_array = ctx.create_array_from(&[bias.len()], bias);+ let mut ids = ctx.create_array_uninitialized(&[t * k], DataType::I32).into_allocation();+ let mut probs = ctx.create_array_uninitialized(&[t * k], T::data_type()).into_allocation();let kernel = <<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, T::data_type()).expect("kernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");kernel.encode(- input_array.buffer().borrow().deref(),- weights_array.buffer().borrow().deref(),- bias_array.buffer().borrow().deref(),- ids_array.buffer().borrow_mut().deref_mut(),- probs_array.buffer().borrow_mut().deref_mut(),+ input_array.allocation(),+ weights_array.allocation(),+ bias_array.allocation(),+ &mut ids,+ &mut probs,t as u32,d_model as u32,e as u32,@@ -51,7 +50,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>();encoder.end_encoding().submit().wait_until_completed().unwrap();- (ids_array.as_slice().to_vec(), probs_array.as_slice().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&ids),+ crate::common::helpers::allocation_to_vec(&probs),+ )}fn run_router_topk_once<B: Backend, T: ArrayElement + Debug + Float>(diff --git a/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rs b/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rsindex 8ef8cc7a1..7cd64b7b3 100644--- a/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/moe/moe_scatter_test.rs@@ -1,10 +1,11 @@-use std::ops::{Deref, DerefMut};-+use half::bf16;+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::StdRng};use backend_uzu::{- Array, ArrayContextExt, ArrayElement, DataType,+ ArrayContextExt, ArrayElement, DataType,backends::{common::{- Backend, Encoder, Kernels,+ Allocation, Backend, Encoder, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,MoeScatterBucketsKernel,@@ -13,9 +14,6 @@ use backend_uzu::{cpu::Cpu,},};-use half::bf16;-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::StdRng};use crate::{common::helpers::create_context, uzu_test};@@ -67,22 +65,22 @@ fn get_output_topk<B: Backend, T: ArrayElement + Float>(d_model: usize,e: usize,k: usize,-) -> (Array<B>, Array<B>) {- let input_array = ctx.create_array_from(&[input.len()], input, "");- let weights_array = ctx.create_array_from(&[weights.len()], weights, "");- let bias_array = ctx.create_array_from(&[bias.len()], bias, "");- let topk_ids_array = ctx.create_array_uninitialized(&[t * k], DataType::I32, "");- let topk_probs_array = ctx.create_array_uninitialized(&[t * k], T::data_type(), "");+) -> (Allocation<B>, Allocation<B>) {+ let input_array = ctx.create_array_from(&[input.len()], input);+ let weights_array = ctx.create_array_from(&[weights.len()], weights);+ let bias_array = ctx.create_array_from(&[bias.len()], bias);+ let mut topk_ids = ctx.create_array_uninitialized(&[t * k], DataType::I32).into_allocation();+ let mut topk_probs = ctx.create_array_uninitialized(&[t * k], T::data_type()).into_allocation();let topk_kernel =<<B as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, T::data_type()).expect("router_topk");let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");topk_kernel.encode(- input_array.buffer().borrow().deref(),- weights_array.buffer().borrow().deref(),- bias_array.buffer().borrow().deref(),- topk_ids_array.buffer().borrow_mut().deref_mut(),- topk_probs_array.buffer().borrow_mut().deref_mut(),+ input_array.allocation(),+ weights_array.allocation(),+ bias_array.allocation(),+ &mut topk_ids,+ &mut topk_probs,t as u32,d_model as u32,e as u32,@@ -92,7 +90,7 @@ fn get_output_topk<B: Backend, T: ArrayElement + Float>();encoder.end_encoding().submit().wait_until_completed().unwrap();- (topk_ids_array, topk_probs_array)+ (topk_ids, topk_probs)}fn get_output<B: Backend, T: ArrayElement + Float>(@@ -103,26 +101,26 @@ fn get_output<B: Backend, T: ArrayElement + Float>(d_model: usize,e: usize,k: usize,-) -> (Vec<i32>, Vec<T>, Array<B>) {+) -> (Vec<i32>, Vec<T>, Vec<u32>) {let ctx = create_context::<B>();// top-k- let (topk_ids_array, topk_probs_array) =+ let (topk_ids, topk_probs) =get_output_topk::<B, T>(ctx.as_ref(), input, weights, bias, t, d_model, e, k);// fused- let offsets_array = ctx.create_array_uninitialized(&[e + 1], DataType::U32, "");- let sumk_array = ctx.create_array_uninitialized(&[1], DataType::U32, "");+ let mut offsets = ctx.create_array_uninitialized(&[e + 1], DataType::U32).into_allocation();+ let mut sumk = ctx.create_array_uninitialized(&[1], DataType::U32).into_allocation();let num_tiles = e.div_ceil(512).max(1);- let partials_array = ctx.create_array_uninitialized(&[num_tiles * 512], DataType::U32, "");+ let mut partials = ctx.create_array_uninitialized(&[num_tiles * 512], DataType::U32).into_allocation();let fused_kernel =<<B as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");fused_kernel.encode(- topk_ids_array.buffer().borrow().deref(),- offsets_array.buffer().borrow_mut().deref_mut(),- sumk_array.buffer().borrow_mut().deref_mut(),- partials_array.buffer().borrow_mut().deref_mut(),+ &topk_ids,+ &mut offsets,+ &mut sumk,+ &mut partials,t as u32,e as u32,k as u32,@@ -134,15 +132,15 @@ fn get_output<B: Backend, T: ArrayElement + Float>(// Partials already created by fused kernel abovelet num_blocks = 1; // Fused kernel uses single blocklet entries = num_blocks * num_tiles * 512usize;- let block_bases_array = ctx.create_array_uninitialized(&[entries], DataType::U32, "");- let block_alloc_array = ctx.create_array_uninitialized(&[entries], DataType::U32, "");+ let mut block_bases = ctx.create_array_uninitialized(&[entries], DataType::U32).into_allocation();+ let mut block_alloc = ctx.create_array_uninitialized(&[entries], DataType::U32).into_allocation();let scatter_bases_kernel = <<B as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(ctx.as_ref()).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel");let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");scatter_bases_kernel.encode(- partials_array.buffer().borrow().deref(),- block_bases_array.buffer().borrow_mut().deref_mut(),- block_alloc_array.buffer().borrow_mut().deref_mut(),+ &partials,+ &mut block_bases,+ &mut block_alloc,e as u32,num_blocks as u32,num_tiles as u32,@@ -150,22 +148,22 @@ fn get_output<B: Backend, T: ArrayElement + Float>(&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- let sumk = unsafe { std::slice::from_raw_parts(sumk_array.cpu_ptr().as_ptr() as *const u32, 1) }[0] as usize;+ let sumk = crate::common::helpers::allocation_to_vec::<B, u32>(&sumk)[0] as usize;// scatterlet scatter_kernel = <<B as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, T::data_type()).expect("Failed to create <<B as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");- let out_ids_array = ctx.create_array_uninitialized(&[sumk], DataType::I32, "");- let out_probs_array = ctx.create_array_uninitialized(&[sumk], T::data_type(), "");+ let mut out_ids = ctx.create_array_uninitialized(&[sumk], DataType::I32).into_allocation();+ let mut out_probs = ctx.create_array_uninitialized(&[sumk], T::data_type()).into_allocation();let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");scatter_kernel.encode(- topk_ids_array.buffer().borrow().deref(),- topk_probs_array.buffer().borrow().deref(),- offsets_array.buffer().borrow().deref(),- block_bases_array.buffer().borrow().deref(),- block_alloc_array.buffer().borrow().deref(),- out_ids_array.buffer().borrow_mut().deref_mut(),- out_probs_array.buffer().borrow_mut().deref_mut(),+ &topk_ids,+ &topk_probs,+ &offsets,+ &block_bases,+ &block_alloc,+ &mut out_ids,+ &mut out_probs,t as u32,e as u32,k as u32,@@ -175,7 +173,11 @@ fn get_output<B: Backend, T: ArrayElement + Float>();encoder.end_encoding().submit().wait_until_completed().unwrap();- (out_ids_array.as_slice().to_vec(), out_probs_array.as_slice().to_vec(), offsets_array)+ (+ crate::common::helpers::allocation_to_vec(&out_ids),+ crate::common::helpers::allocation_to_vec(&out_probs),+ crate::common::helpers::allocation_to_vec(&offsets),+ )}fn test_scatter_internal<B: Backend, T: ArrayElement + Float>(@@ -207,10 +209,10 @@ fn test_scatter_internal<B: Backend, T: ArrayElement + Float>(e,k,);- let (cpu_ids, _cpu_probs, offsets_cpu) =- cpu_expert_buckets(topk_ids_cpu.as_slice(), &topk_probs_cpu.as_slice::<T>(), t, e, k);- let offsets_gpu = unsafe { std::slice::from_raw_parts(out_offsets_array.cpu_ptr().as_ptr() as *const u32, e + 1) };- assert_eq!(offsets_gpu, &offsets_cpu[..]);+ let cpu_topk_ids: Vec<i32> = crate::common::helpers::allocation_to_vec(&topk_ids_cpu);+ let cpu_topk_probs: Vec<T> = crate::common::helpers::allocation_to_vec(&topk_probs_cpu);+ let (cpu_ids, _cpu_probs, offsets_cpu) = cpu_expert_buckets(&cpu_topk_ids, &cpu_topk_probs, t, e, k);+ assert_eq!(out_offsets_array, offsets_cpu);// Compare per-expert multisets of idsfor ei in 0..e {diff --git a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rs b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rsindex 249482753..052f29639 100644--- a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_cls_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -50,13 +49,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;let output_len = (input.batch_size * input.hidden_dim) as usize;- let input_array = context.create_array_from(&[input_len], &input.input, "");- let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");+ let input_array = context.create_array_from(&[input_len], &input.input);+ let mut output = context+ .create_array_uninitialized(&[output_len], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- input_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ input_array.allocation(),+ &mut output,input.seq_len,input.hidden_dim,input.batch_size,@@ -64,7 +65,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_internal<T: ArrayElement + Float + Debug + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rs b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rsindex b6c1b8104..bb93185a0 100644--- a/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/pooling/pooling_mean_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -50,13 +49,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let input_len = (input.batch_size * input.seq_len * input.hidden_dim) as usize;let output_len = (input.batch_size * input.hidden_dim) as usize;- let input_array = context.create_array_from(&[input_len], &input.input, "");- let output_array = context.create_array_uninitialized(&[output_len], T::data_type(), "");+ let input_array = context.create_array_from(&[input_len], &input.input);+ let mut output = context+ .create_array_uninitialized(&[output_len], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- input_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ input_array.allocation(),+ &mut output,input.seq_len,input.hidden_dim,input.batch_size,@@ -64,7 +65,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_internal<T: ArrayElement + Float + Debug + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rsindex 7c5c3c437..ca8b3526d 100644--- a/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/qk_norm_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -126,14 +125,14 @@ fn get_output<.expect("Failed to create QKNormKernel");let qkv_len = input.qkv.len();- let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");- let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");+ let mut qkv = context.create_array_from(&[qkv_len], &input.qkv).into_allocation();+ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- None::<&B::DenseBuffer>,- scales_array.buffer().borrow().deref(),- qkv_array.buffer().borrow_mut().deref_mut(),+ None::<&backend_uzu::backends::common::Allocation<B>>,+ scales_array.allocation(),+ &mut qkv,input.batch_size,input.num_q_heads,input.num_kv_heads,@@ -147,7 +146,7 @@ fn get_output<);encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");- qkv_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&qkv)}fn test_internal<diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rsindex 0168c0ae7..471d15b50 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs@@ -3,7 +3,7 @@ use std::fmt::{Debug, Display};use backend_uzu::{ArrayElement, DataType,backends::{- common::{Backend, Context, DenseBuffer, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedKernel},+ common::{Allocation, Backend, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedKernel},cpu::Cpu,},};@@ -15,7 +15,10 @@ use rand::{RngExt, SeedableRng, rngs::SmallRng};use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};use crate::{- common::{helpers::alloc_buffer_with_data, type_short_name},+ common::{+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ type_short_name,+ },uzu_bench, uzu_test,};@@ -191,16 +194,15 @@ fn get_output<B: Backend, T: ArrayElement + Float>().expect("Failed to create QuantizedMatmulQmmTransposedKernel");- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);+ let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));- let had_buf = hadamard_factors.map(|f| alloc_buffer_with_data::<B, i32>(&context, f));+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));+ let had_buf = hadamard_factors.map(|f| alloc_allocation_with_data::<B, i32>(&context, f));- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");+ let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);+ let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(@@ -219,9 +221,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;- let y_len = (input.m as usize) * (input.n as usize);- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+ allocation_to_vec(&y_buf)}fn check_against_expected<T: ArrayElement + Float + Debug + Display>(@@ -660,14 +660,9 @@ fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>() {// Use the dispatcher-selected tile configs: BM=8 small tile for M<48, big tile for M>=48.// We always use the BM=8 small tile kernel for M in 4..=47 to match the dispatch threshold.- let block_size: usize = if bits == 4 {- 512- } else {- 256- };+ let block_size: usize = if bits == 4 { 512 } else { 256 };- for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336])- {+ for (m, n, k) in iproduct!([4usize, 5, 6, 7, 8, 16, 32, 48, 64], [2048usize, 4096, 14336], [2048usize, 4096, 14336]) {if n % 32 != 0 || k % block_size != 0 {continue;}@@ -699,33 +694,29 @@ fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>().unwrap();- let w_buf = alloc_buffer_with_data::<B, u32>(+ let w_buf = alloc_allocation_with_data::<B, u32>(context,&(0..n * k * bits as usize / 32).map(|_| rng.random_range(0..u32::MAX)).collect::<Vec<_>>(),);- let scales_buf = alloc_buffer_with_data::<B, T>(+ let scales_buf = alloc_allocation_with_data::<B, T>(context,&(0..n * num_groups).map(|_| T::from(rng.random_range(0.01f32..1.0f32)).unwrap()).collect::<Vec<_>>(),);- let x_buf = alloc_buffer_with_data::<B, T>(+ let x_buf = alloc_allocation_with_data::<B, T>(context,&(0..m * k).map(|_| T::from(rng.random_range(-1.0f32..1.0f32)).unwrap()).collect::<Vec<_>>(),);- let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);- let zp_stride = if bits == 4 {- (num_groups + 1) / 2- } else {- num_groups- };+ let zp_stride = if bits == 4 { (num_groups + 1) / 2 } else { num_groups };let zp_buf = use_zero_points.then(|| {- alloc_buffer_with_data::<B, u8>(+ alloc_allocation_with_data::<B, u8>(context,&(0..n * zp_stride).map(|_| rng.random_range(0u8..u8::MAX)).collect::<Vec<_>>(),)});let bias_buf = use_mlx_quant.then(|| {- alloc_buffer_with_data::<B, T>(+ alloc_allocation_with_data::<B, T>(context,&(0..n * num_groups).map(|_| T::from(rng.random_range(-0.5f32..0.5f32)).unwrap()).collect::<Vec<_>>(),)@@ -744,7 +735,7 @@ fn bench_qmm_transposed_typed<B: Backend, T: ArrayElement + Float>(bias_buf.as_ref(),&x_buf,&mut y_buf,- None::<&B::DenseBuffer>,+ None::<&Allocation<B>>,k as u32,n as u32,m as u32,diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex 034482e75..c2974c4d7 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -4,7 +4,7 @@ use backend_uzu::{ArrayElement, DataType,backends::{common::{- Backend, Context, DenseBuffer, Encoder, Kernels,+ Allocation, Backend, Context, Encoder, Kernels,kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},},cpu::Cpu,@@ -17,20 +17,22 @@ use rand::{RngExt, SeedableRng, rngs::SmallRng};use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};use crate::{- common::{helpers::alloc_buffer_with_data, type_short_name},+ common::{+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ type_short_name,+ },uzu_bench, uzu_test,};fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");- let w_buf = alloc_buffer_with_data::<Cpu, u32>(&context, &input.w_packed);- let scales_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.scales);- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<Cpu, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<Cpu, T>(&context, b));- let x_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.x);- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");+ let w_buf = alloc_allocation_with_data::<Cpu, u32>(&context, &input.w_packed);+ let scales_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.scales);+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<Cpu, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<Cpu, T>(&context, b));+ let x_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.x);+ let mut y_buf = alloc_allocation::<Cpu, T>(&context, (input.m as usize) * (input.n as usize));let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -58,21 +60,18 @@ fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;- let y_len = (input.m as usize) * (input.n as usize);- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+ allocation_to_vec(&y_buf)}fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");+ let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));+ let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);+ let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -93,7 +92,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {bias_buf.as_ref(),&x_buf,&mut y_buf,- None::<&B::DenseBuffer>,+ None::<&Allocation<B>>,input.k,input.n,input.m,@@ -102,9 +101,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;- let y_len = (input.m as usize) * (input.n as usize);- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+ allocation_to_vec(&y_buf)}fn get_test_data<T: ArrayElement + Float>(@@ -347,25 +344,25 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>().unwrap();- let w_buf = alloc_buffer_with_data::<B, u32>(+ let w_buf = alloc_allocation_with_data::<B, u32>(context,&gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),);- let scales_buf = alloc_buffer_with_data::<B, T>(+ let scales_buf = alloc_allocation_with_data::<B, T>(context,&gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>(),);- let x_buf = alloc_buffer_with_data::<B, T>(+ let x_buf = alloc_allocation_with_data::<B, T>(context,&gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>(),);- let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();+ let mut y_buf = alloc_allocation::<B, T>(context, m * n);let zp_buf = use_zero_points.then(|| {let zp_stride = if bits == 4 {@@ -373,10 +370,10 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(} else {num_groups};- alloc_buffer_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))+ alloc_allocation_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))});let bias_buf = use_mlx_quant.then(|| {- alloc_buffer_with_data::<B, T>(+ alloc_allocation_with_data::<B, T>(context,&gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups).iter()@@ -397,7 +394,7 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(bias_buf.as_ref(),&x_buf,&mut y_buf,- None::<&B::DenseBuffer>,+ None::<&Allocation<B>>,k as u32,n as u32,m as u32,diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rsindex fd448a593..80aa4f71d 100644--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs@@ -3,27 +3,29 @@ use std::fmt::{Debug, Display};use backend_uzu::{ArrayElement, DataType,backends::{- common::{Backend, Context, DenseBuffer, Encoder, Kernels, kernel::QuantizedMatmulQmvKernel},+ common::{Backend, Context, Encoder, Kernels, kernel::QuantizedMatmulQmvKernel},cpu::Cpu,},};use num_traits::Float;use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};+use crate::{+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ uzu_test,+};fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = B::Context::new().expect("Failed to create Context");- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);+ let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");+ let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);+ let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");@@ -51,9 +53,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;- let y_len = (input.m as usize) * (input.n as usize);- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+ allocation_to_vec(&y_buf)}/// Create test data for a qmv scenario: m×k * k×n = m×ndiff --git a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rsindex 3cd5d2648..63bb22b79 100644--- a/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/rms_norm_test.rs@@ -1,13 +1,12 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},time::{Duration, Instant},};use backend_uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{Backend, Context, DenseBuffer, Encoder, Kernels, kernel::RMSNormKernel},+ common::{Allocation, Context, Encoder, Kernels, kernel::RMSNormKernel, Backend},cpu::Cpu,},};@@ -138,25 +137,23 @@ fn get_output<.expect("Failed to create RMSNormKernel");let input_size = input.input.len();- let input_array = context.create_array_from(&[input_size], &input.input, "");- let input_array_buffer_rc = input_array.buffer();- let input_array_borrow = input_array_buffer_rc.borrow();- let input_array_deref = input_array_borrow.deref();- let input_buffer = (!input.in_place).then(|| input_array_deref);-- let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");- let output_array = match input.in_place {- true => context.create_array_from(&[input_size], &input.output, ""),- false => context.create_array_uninitialized(&[input_size], OutputT::data_type(), ""),+ let input_buffer = (!input.in_place).then(|| context.create_array_from(&[input_size], &input.input).into_allocation());++ let scales_array = context.create_array_from(&[input.scales.len()], &input.scales);+ let mut output = match input.in_place {+ true => context.create_array_from(&[input_size], &input.output).into_allocation(),+ false => context+ .create_array_uninitialized(&[input_size], OutputT::data_type())+ .into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- input_buffer,- scales_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),- None::<&mut B::DenseBuffer>,- None::<&B::DenseBuffer>,+ input_buffer.as_ref(),+ scales_array.allocation(),+ &mut output,+ None::<&mut Allocation<B>>,+ None::<&Allocation<B>>,input.batch_size,input.element_count,input.epsilon,@@ -169,7 +166,11 @@ fn get_output<let host_elapsed_ms = instant.elapsed().as_secs_f64() * 1e3;let gpu_elapsed_ms = completed.gpu_execution_time();- (output_array.as_slice().to_vec(), host_elapsed_ms, gpu_elapsed_ms)+ (+ crate::common::helpers::allocation_to_vec(&output),+ host_elapsed_ms,+ gpu_elapsed_ms,+ )}fn test_internal<@@ -430,19 +431,9 @@ fn bench_rms_norm(c: &mut Criterion) {let (input_data, scale_data) = get_rms_norm_data(1337, batch_size, model_dim);let input_size = batch_size * model_dim;- let input_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();- unsafe {- std::slice::from_raw_parts_mut::<T>(input_buffer.cpu_ptr().as_ptr() as *mut T, input_size)- .copy_from_slice(input_data.as_ref());- }-- let scales_buffer = context.create_buffer(model_dim * std::mem::size_of::<T>()).unwrap();- unsafe {- std::slice::from_raw_parts_mut::<T>(scales_buffer.cpu_ptr().as_ptr() as *mut T, model_dim)- .copy_from_slice(scale_data.as_ref());- }-- let mut output_buffer = context.create_buffer(input_size * std::mem::size_of::<T>()).unwrap();+ let input_buffer = context.create_array_from(&[input_size], input_data.as_ref()).into_allocation();+ let scales_buffer = context.create_array_from(&[model_dim], scale_data.as_ref()).into_allocation();+ let mut output_buffer = context.create_array_uninitialized(&[input_size], T::data_type()).into_allocation();group.throughput(Throughput::Elements((batch_size * model_dim) as u64));@@ -455,8 +446,8 @@ fn bench_rms_norm(c: &mut Criterion) {Some(&input_buffer),&scales_buffer,&mut output_buffer,- None::<&mut <B as Backend>::DenseBuffer>,- None::<&<B as Backend>::DenseBuffer>,+ None::<&mut Allocation<B>>,+ None::<&Allocation<B>>,batch_size as u32,model_dim as u32,epsilon,diff --git a/crates/backend-uzu/tests/unit/kernel/rope_test.rs b/crates/backend-uzu/tests/unit/kernel/rope_test.rsindex 013212eab..e30d780f6 100644--- a/crates/backend-uzu/tests/unit/kernel/rope_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/rope_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -83,21 +82,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let queries_len = input.num_heads as usize * input.suffix_length as usize * input.head_dim as usize;let keys_len = input.num_groups as usize * input.suffix_length as usize * input.head_dim as usize;- let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");- let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines, "");- let sines_array = context.create_array_from(&[cos_sin_len], &input.sines, "");- let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions, "");- let rotated_queries_array = context.create_array_uninitialized(&[queries_len], T::data_type(), "");- let rotated_keys_array = context.create_array_uninitialized(&[keys_len], T::data_type(), "");+ let qkv_array = context.create_array_from(&[qkv_len], &input.qkv);+ let cosines_array = context.create_array_from(&[cos_sin_len], &input.cosines);+ let sines_array = context.create_array_from(&[cos_sin_len], &input.sines);+ let token_positions_array = context.create_array_from(&[input.suffix_length as usize], &input.token_positions);+ let mut rotated_queries = context+ .create_array_uninitialized(&[queries_len], T::data_type())+ .into_allocation();+ let mut rotated_keys = context+ .create_array_uninitialized(&[keys_len], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- qkv_array.buffer().borrow().deref(),- cosines_array.buffer().borrow().deref(),- sines_array.buffer().borrow().deref(),- token_positions_array.buffer().borrow().deref(),- rotated_queries_array.buffer().borrow_mut().deref_mut(),- rotated_keys_array.buffer().borrow_mut().deref_mut(),+ qkv_array.allocation(),+ cosines_array.allocation(),+ sines_array.allocation(),+ token_positions_array.allocation(),+ &mut rotated_queries,+ &mut rotated_keys,input.head_dim,input.rope_dim,input.num_heads,@@ -108,7 +111,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,);encoder.end_encoding().submit().wait_until_completed().unwrap();- (rotated_queries_array.as_slice().to_vec(), rotated_keys_array.as_slice().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&rotated_queries),+ crate::common::helpers::allocation_to_vec(&rotated_keys),+ )}fn test_internal<T: ArrayElement + Float + Debug + Display>(input: &Input<T>) {diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rs b/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rsindex 2706fee92..3790951f6 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/argmax_test_old.rs@@ -1,7 +1,6 @@use std::{fmt::Display,mem::size_of,- ops::{Deref, DerefMut},};use backend_uzu::{@@ -69,13 +68,13 @@ fn get_output_single<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> V.expect("Failed to create ArgmaxSingleKernel");let len = (input.batch_size * input.vocab_size) as usize;- let logits_array = context.create_array_from(&[len], &input.logits, "");- let output_array = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32, "");+ let logits_array = context.create_array_from(&[len], &input.logits);+ let mut output = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ logits_array.allocation(),+ &mut output,input.batch_size,input.vocab_size,&mut encoder,@@ -83,7 +82,7 @@ fn get_output_single<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vencoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice::<u32>().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<u32> {@@ -95,29 +94,29 @@ fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) ->.expect("Failed to create ArgmaxFinalKernel");let len = (input.batch_size * input.vocab_size) as usize;- let logits_array = context.create_array_from(&[len], &input.logits, "");- let output_array = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32, "");+ let logits_array = context.create_array_from(&[len], &input.logits);+ let mut output = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32).into_allocation();let block_size = 1024;let grain_size = 4;let elements_per_group = block_size * grain_size;let vocab_groups_per_batch = (input.vocab_size as usize + elements_per_group - 1) / elements_per_group;let partial_results_count = input.batch_size as usize * vocab_groups_per_batch;- let mut partial_results_buffer = context- .create_buffer(partial_results_count * size_of::<ArgmaxPair>())- .expect("Failed to create partial results buffer");+ let mut partial_results = context+ .create_array_uninitialized(&[partial_results_count * size_of::<ArgmaxPair>()], DataType::U8)+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");main_kernel.encode(- logits_array.buffer().borrow().deref(),- &mut partial_results_buffer,+ logits_array.allocation(),+ &mut partial_results,input.batch_size,input.vocab_size,&mut encoder,);final_kernel.encode(- &partial_results_buffer,- output_array.buffer().borrow_mut().deref_mut(),+ &partial_results,+ &mut output,input.batch_size,input.vocab_size,&mut encoder,@@ -125,7 +124,7 @@ fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) ->encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice::<u32>().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_single_pass<T: ArrayElement + Float + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rsindex f1fbc901f..3a227373e 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/bitmask_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -133,29 +132,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let len = (input.batch_size * input.vocab_size) as usize;let bitmask_len = input.bitmask.len();- let logits_array = context.create_array_from(&[len], &input.logits, "");- let bitmask_array = context.create_array_from(&[bitmask_len], &input.bitmask, "");- let logits_array_buffer_rc = logits_array.buffer();- let logits_array_borrow = logits_array_buffer_rc.borrow();- let logits_array_deref = logits_array_borrow.deref();- let logits_buffer = (!input.in_place).then(|| logits_array_deref);- let output_array = match input.in_place {- true => context.create_array_from(&[len], &input.logits, ""),- false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());+ let bitmask_array = context.create_array_from(&[bitmask_len], &input.bitmask);+ let mut output = match input.in_place {+ true => context.create_array_from(&[len], &input.logits).into_allocation(),+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_buffer,- bitmask_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ logits_buffer.as_ref(),+ bitmask_array.allocation(),+ &mut output,input.batch_size,input.vocab_size,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_basic<T: ArrayElement + Float + Debug + Display>(in_place: bool) {diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rsindex 84e1b2b0d..c51363f0c 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/gumbel_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -57,25 +56,21 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.expect("Failed to create GumbelKernel");let len = (input.batch_size * input.vocab_size) as usize;- let logits_array = context.create_array_from(&[len], &input.logits, "");- let logits_array_buffer_rc = logits_array.buffer();- let logits_array_borrow = logits_array_buffer_rc.borrow();- let logits_array_deref = logits_array_borrow.deref();- let logits_buffer = (!input.in_place).then(|| logits_array_deref);+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());- let seeds_array = context.create_array_from(&[input.batch_size as usize], &input.seeds, "");+ let seeds_array = context.create_array_from(&[input.batch_size as usize], &input.seeds);- let output_array = match input.in_place {- true => context.create_array_from(&[len], &input.logits, ""),- false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ let mut output = match input.in_place {+ true => context.create_array_from(&[len], &input.logits).into_allocation(),+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_buffer,- seeds_array.buffer().borrow().deref(),+ logits_buffer.as_ref(),+ seeds_array.allocation(),0,- output_array.buffer().borrow_mut().deref_mut(),+ &mut output,input.batch_size,input.vocab_size,&mut encoder,@@ -83,7 +78,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}// Verify determinism: same seeds produce same output.diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rsindex 6f31a8411..f6d37afbd 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/min_p_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -51,20 +50,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.expect("Failed to create MinPKernel");let len = (input.batch_size * input.vocab_size) as usize;- let logits_array = context.create_array_from(&[len], &input.logits, "");- let logits_array_buffer_rc = logits_array.buffer();- let logits_array_borrow = logits_array_buffer_rc.borrow();- let logits_array_deref = logits_array_borrow.deref();- let logits_buffer = (!input.in_place).then(|| logits_array_deref);- let output_array = match input.in_place {- true => context.create_array_from(&[len], &input.logits, ""),- false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());+ let mut output = match input.in_place {+ true => context.create_array_from(&[len], &input.logits).into_allocation(),+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_buffer,- output_array.buffer().borrow_mut().deref_mut(),+ logits_buffer.as_ref(),+ &mut output,input.batch_size,input.vocab_size,input.min_p,@@ -72,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_internal<T: ArrayElement + Float + Debug + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rsindex 544d32109..1ee3ad552 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/temperature_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -55,20 +54,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.expect("Failed to create TemperatureKernel");let len = (input.batch_size * input.vocab_size) as usize;- let logits_array = context.create_array_from(&[len], &input.logits, "");- let logits_array_buffer_rc = logits_array.buffer();- let logits_array_borrow = logits_array_buffer_rc.borrow();- let logits_array_deref = logits_array_borrow.deref();- let logits_buffer = (!input.in_place).then(|| logits_array_deref);- let output_array = match input.in_place {- true => context.create_array_from(&[len], &input.logits, ""),- false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ let input_logits = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());+ let mut output = match input.in_place {+ true => context.create_array_from(&[len], &input.logits).into_allocation(),+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_buffer,- output_array.buffer().borrow_mut().deref_mut(),+ input_logits.as_ref(),+ &mut output,input.batch_size,input.vocab_size,input.temperature,@@ -77,7 +72,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn test_internal<T: ArrayElement + Float + Debug + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rsindex 597013648..07700d020 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/test_argmax.rs@@ -1,18 +1,18 @@-use std::mem::MaybeUninit;+use std::mem::{MaybeUninit, size_of};+use criterion::{BenchmarkId, Criterion, Throughput};+use num_traits::{Float, NumCast};+use proptest::prelude::*;+use rand::{RngExt, SeedableRng, rngs::SmallRng};+use rand_distr::Normal;use backend_uzu::{- ArrayElement,+ ArrayContextExt, ArrayElement, DataType,backends::common::{- Backend, Context, DenseBuffer, Encoder, Kernels,+ AllocationType, Backend, Context, Encoder, Kernels,gpu_types::ArgmaxPair,kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},},};-use criterion::{BenchmarkId, Criterion, Throughput};-use num_traits::{Float, NumCast};-use proptest::prelude::*;-use rand::{RngExt, SeedableRng, rngs::SmallRng};-use rand_distr::Normal;use crate::{common::{@@ -43,17 +43,15 @@ fn do_argmax_backend<B: Backend, T: ArrayElement + Float>(batch_size: usize,vocab_size: usize,) -> Result<ArgmaxTestResults, TestCaseError> {- let logits_buffer = context.create_buffer(logits.len() * std::mem::size_of::<T>()).unwrap();- unsafe {- std::slice::from_raw_parts_mut::<T>(logits_buffer.cpu_ptr().as_ptr() as *mut T, logits.len())- .copy_from_slice(logits);- }-- let mut twopass_partial_results_buffer =- context.create_buffer(batch_size * vocab_size.div_ceil(4096) * std::mem::size_of::<ArgmaxPair>()).unwrap();-- let mut single_output_buffer = context.create_buffer(batch_size * 4).unwrap();- let mut twopass_output_buffer = context.create_buffer(batch_size * 4).unwrap();+ let logits_buffer = context.create_array_from(&[logits.len()], logits).into_allocation();+ let mut twopass_partial_results_buffer = context+ .create_allocation(+ batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),+ AllocationType::Global,+ )+ .unwrap();+ let mut single_output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();+ let mut twopass_output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();let single_kernel = <B::Kernels as Kernels>::ArgmaxSingleKernel::new(context, T::data_type()).unwrap();let twopass_kernel_main = <B::Kernels as Kernels>::ArgmaxMainKernel::new(context, T::data_type()).unwrap();@@ -77,23 +75,12 @@ fn do_argmax_backend<B: Backend, T: ArrayElement + Float>();encoder.end_encoding().submit().wait_until_completed().unwrap();- let single_output = unsafe {- std::slice::from_raw_parts(single_output_buffer.cpu_ptr().as_ptr() as *const u32, batch_size)- .iter()- .copied()- .collect()- };-- let twopass_output = unsafe {- std::slice::from_raw_parts(twopass_output_buffer.cpu_ptr().as_ptr() as *const u32, batch_size)- .iter()- .copied()- .collect()- };+ let single_output = crate::common::helpers::allocation_to_vec::<B, u32>(&single_output_buffer);+ let twopass_output = crate::common::helpers::allocation_to_vec::<B, u32>(&twopass_output_buffer);prop_assert_eq!(&single_output, &twopass_output, "single and twopass argmax output differs");- Ok(ArgmaxTestResults(single_output))+ Ok(ArgmaxTestResults(single_output.into_boxed_slice()))}fn get_argmax_data<T: ArrayElement + Float>(@@ -143,7 +130,7 @@ fn bench_argmax(c: &mut Criterion) {<<B as Backend>::Kernels as Kernels>::ArgmaxMainKernel::new(&context, T::data_type()).unwrap();let twopass_kernel_final = <<B as Backend>::Kernels as Kernels>::ArgmaxFinalKernel::new(&context).unwrap();- let mut output_buffer = context.create_buffer(batch_size * std::mem::size_of::<u32>()).unwrap();+ let mut output_buffer = context.create_array_uninitialized(&[batch_size], DataType::U32).into_allocation();let mut group = c.benchmark_group(format!("{}/Kernel/Sampling/Argmax", type_short_name::<B>()));@@ -154,14 +141,12 @@ fn bench_argmax(c: &mut Criterion) {262144, // Gemma-3-1b] {let logits_data = get_argmax_data::<T>(1337, batch_size, vocab_size);- let logits_buffer = context.create_buffer(logits_data.len() * std::mem::size_of::<T>()).unwrap();- unsafe {- std::slice::from_raw_parts_mut::<T>(logits_buffer.cpu_ptr().as_ptr() as *mut T, logits_data.len())- .copy_from_slice(logits_data.as_ref());- }-+ let logits_buffer = context.create_array_from(&[logits_data.len()], logits_data.as_ref()).into_allocation();let mut twopass_partial_results_buffer = context- .create_buffer(batch_size * vocab_size.div_ceil(4096) * std::mem::size_of::<ArgmaxPair>())+ .create_allocation(+ batch_size * vocab_size.div_ceil(4096) * size_of::<ArgmaxPair>(),+ AllocationType::Global,+ ).unwrap();group.throughput(Throughput::Elements(vocab_size as u64));diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rsindex 963784616..d5c800731 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/top_k_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -57,20 +56,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.expect("Failed to create TopKKernel");let len = (input.batch_size * input.vocab_size) as usize;- let logits_array = context.create_array_from(&[len], &input.logits, "");- let logits_array_buffer_rc = logits_array.buffer();- let logits_array_borrow = logits_array_buffer_rc.borrow();- let logits_array_deref = logits_array_borrow.deref();- let logits_buffer = (!input.in_place).then(|| logits_array_deref);- let output_array = match input.in_place {- true => context.create_array_from(&[len], &input.logits, ""),- false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());+ let mut output = match input.in_place {+ true => context.create_array_from(&[len], &input.logits).into_allocation(),+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_buffer,- output_array.buffer().borrow_mut().deref_mut(),+ logits_buffer.as_ref(),+ &mut output,input.batch_size,input.vocab_size,input.top_k,@@ -79,7 +74,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn assert_top_k_equal<T: ArrayElement + Float + Display>(diff --git a/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rsindex ceb31bb3a..a5dbf16da 100644--- a/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/sampling/top_p_test.rs@@ -1,25 +1,27 @@use std::{collections::HashSet,fmt::{Debug, Display},- ops::{Deref, DerefMut},};+use half::{bf16, f16};+use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::StdRng, seq::SliceRandom};use backend_uzu::{ArrayContextExt, ArrayElement, DataType,backends::{common::{- Backend, Context, Encoder, Kernels,+ AllocationType, Backend, Context, Encoder, Kernels,kernel::{TopPKernel, sampling::SamplingKernel},},cpu::Cpu,},session::parameter::{SamplingMethod, SamplingProcessingOrder},};-use half::{bf16, f16};-use num_traits::Float;-use rand::{RngExt, SeedableRng, rngs::StdRng, seq::SliceRandom};-use crate::uzu_test;+use crate::{+ common::helpers::{alloc_allocation_with_data, allocation_to_vec},+ uzu_test,+};const TEST_SAMPLING_SEED: u64 = 42;@@ -64,20 +66,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.expect("Failed to create TopPKernel");let len = (input.batch_size * input.vocab_size) as usize;- let logits_array = context.create_array_from(&[len], &input.logits, "");- let logits_array_buffer_rc = logits_array.buffer();- let logits_array_borrow = logits_array_buffer_rc.borrow();- let logits_array_deref = logits_array_borrow.deref();- let logits_buffer = (!input.in_place).then(|| logits_array_deref);- let output_array = match input.in_place {- true => context.create_array_from(&[len], &input.logits, ""),- false => context.create_array_uninitialized(&[len], T::data_type(), ""),+ let logits_buffer = (!input.in_place).then(|| context.create_array_from(&[len], &input.logits).into_allocation());+ let mut output = match input.in_place {+ true => context.create_array_from(&[len], &input.logits).into_allocation(),+ false => context.create_array_uninitialized(&[len], T::data_type()).into_allocation(),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_buffer,- output_array.buffer().borrow_mut().deref_mut(),+ logits_buffer.as_ref(),+ &mut output,input.batch_size,input.vocab_size,input.top_p,@@ -86,7 +84,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&output)}fn assert_top_p_equal<T: ArrayElement + Float + Display>(@@ -239,8 +237,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(let p = (k as f32) * 0.1;- let kernel = SamplingKernel::<B>::new(&context, DataType::F32, batch_size, vocab_size)- .expect("Failed to create sampling kernel");+ let kernel = SamplingKernel::<B>::new(&context, DataType::F32).expect("Failed to create sampling kernel");// Build probability tablelet low_prob = (1.0 - p) / (vocab_size - k) as f32;@@ -271,25 +268,25 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(}).collect();- let output_array = context.create_array_uninitialized(&[batch_size], DataType::U32, "");+ let mut output_allocation = context+ .create_allocation(batch_size * std::mem::size_of::<u32>(), AllocationType::Global)+ .expect("Failed to create allocation");let num_samples = 1000;let mut counter = vec![0i32; batch_size * vocab_size];for draw in 0..num_samples {- let logits_array = context.create_array_from(&[batch_size * vocab_size], &logits, "");let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];- let seeds_array = context.create_array_from(&[batch_size], &seeds, "");+ let mut logits_allocation = alloc_allocation_with_data::<B, f32>(&context, &logits);+ let seeds_allocation = alloc_allocation_with_data::<B, u64>(&context, &seeds);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_array.buffer().borrow_mut().deref_mut(),- seeds_array.buffer().borrow().deref(),- 0,+ &mut logits_allocation,+ &seeds_allocation,None,- 0,- output_array.buffer().borrow_mut().deref_mut(),+ &mut output_allocation,SamplingMethod::Stochastic {temperature: None,top_k: None,@@ -304,7 +301,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(.expect("encode");encoder.end_encoding().submit().wait_until_completed().unwrap();- let sampled_ids: &[u32] = output_array.as_slice();+ let sampled_ids = allocation_to_vec::<B, u32>(&output_allocation);for (i, &sampled_id) in sampled_ids.iter().enumerate() {assert!((sampled_id as usize) < vocab_size, "Sampled token out of range");@@ -361,7 +358,7 @@ fn test_topp_sampling_statistical_large() {let context = <B as Backend>::Context::new().expect("Failed to create Context");let kernel =- SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");+ SamplingKernel::<B>::new(&context, DataType::F32).expect("Failed to create sampling kernel");let mut rng = StdRng::seed_from_u64(42);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -377,24 +374,24 @@ fn test_topp_sampling_statistical_large() {probs[b * VOCAB..(b + 1) * VOCAB].copy_from_slice(&dist);}- let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");+ let mut output_allocation = context+ .create_allocation(BATCH * std::mem::size_of::<u32>(), AllocationType::Global)+ .expect("Failed to create allocation");let mut counters = vec![0u32; BATCH * VOCAB];for draw in 0..NUM_DRAWS {- let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];- let seeds_array = context.create_array_from(&[BATCH], &seeds, "");+ let mut logits_allocation = alloc_allocation_with_data::<B, f32>(&context, &logits);+ let seeds_allocation = alloc_allocation_with_data::<B, u64>(&context, &seeds);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_array.buffer().borrow_mut().deref_mut(),- seeds_array.buffer().borrow().deref(),- 0,+ &mut logits_allocation,+ &seeds_allocation,None,- 0,- output_array.buffer().borrow_mut().deref_mut(),+ &mut output_allocation,SamplingMethod::Stochastic {temperature: None,top_k: None,@@ -409,7 +406,7 @@ fn test_topp_sampling_statistical_large() {.expect("encode");encoder.end_encoding().submit().wait_until_completed().unwrap();- let sample_ids: &[u32] = output_array.as_slice();+ let sample_ids = allocation_to_vec::<B, u32>(&output_allocation);for (b, &tok) in sample_ids.iter().enumerate() {counters[b * VOCAB + tok as usize] += 1;}@@ -449,7 +446,7 @@ fn perf_topp_128k_vocab() {let context = <B as Backend>::Context::new().expect("Failed to create Context");let kernel =- SamplingKernel::<B>::new(&context, DataType::F32, BATCH, VOCAB).expect("Failed to create sampling kernel");+ SamplingKernel::<B>::new(&context, DataType::F32).expect("Failed to create sampling kernel");let mut rng = StdRng::seed_from_u64(123);let mut logits = vec![0.0f32; BATCH * VOCAB];@@ -457,21 +454,21 @@ fn perf_topp_128k_vocab() {*x = rng.random_range(-6.0f32..6.0f32);}- let logits_array = context.create_array_from(&[BATCH * VOCAB], &logits, "");let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED; BATCH];- let seeds_array = context.create_array_from(&[BATCH], &seeds, "");- let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");+ let mut logits_allocation = alloc_allocation_with_data::<B, f32>(&context, &logits);+ let seeds_allocation = alloc_allocation_with_data::<B, u64>(&context, &seeds);+ let mut output_allocation = context+ .create_allocation(BATCH * std::mem::size_of::<u32>(), AllocationType::Global)+ .expect("Failed to create allocation");let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- logits_array.buffer().borrow_mut().deref_mut(),- seeds_array.buffer().borrow().deref(),- 0,+ &mut logits_allocation,+ &seeds_allocation,None,- 0,- output_array.buffer().borrow_mut().deref_mut(),+ &mut output_allocation,SamplingMethod::Stochastic {temperature: None,top_k: None,@@ -499,8 +496,8 @@ fn perf_topp_128k_vocab() {host_elapsed_ms);- let sample_ids: &[u32] = output_array.as_slice();- for &tok in sample_ids {+ let sample_ids = allocation_to_vec::<B, u32>(&output_allocation);+ for tok in sample_ids {assert!((tok as usize) < VOCAB, "Sampled id out of range");}});diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rsindex 343ca94f5..47da37472 100644--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_decode_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -42,46 +41,35 @@ fn get_output<T: ArrayElement + Float, B: Backend>().expect("Failed to create ShortConvDecodeKernel");- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");- let w_array = context.create_array_from(&[input.w.len()], &input.w, "");- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, ""));+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);+ let w_array = context.create_array_from(&[input.w.len()], &input.w);+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));let out_size = input.suffix_len as usize * input.model_dim as usize;- let out_array = context.create_array_uninitialized(&[out_size], T::data_type(), "");+ let mut out = context.create_array_uninitialized(&[out_size], T::data_type()).into_allocation();let state_size = input.model_dim as usize * input.state_stride as usize;- let next_state_array = if state_in_place {- if state_size > 0 {- context.create_array_from(&[state_size], &input.state, "")- } else {- context.create_array_uninitialized(&[1], T::data_type(), "")- }- } else {- context.create_array_uninitialized(&[state_size.max(1)], T::data_type(), "")- };+ let state_allocation_size = state_size.max(1);+ let state_data: Vec<T> =+ input.state.iter().copied().chain(std::iter::repeat(T::zero())).take(state_allocation_size).collect();- let state_array = if state_in_place {- None- } else if state_size > 0 {- Some(context.create_array_from(&[state_size], &input.state, ""))+ let mut next_state = if state_in_place {+ context.create_array_from(&[state_allocation_size], &state_data).into_allocation()} else {- Some(context.create_array_uninitialized(&[1], T::data_type(), ""))+ context.create_array_uninitialized(&[state_allocation_size], T::data_type()).into_allocation()};- let bias_buf_rc = b_array.as_ref().map(|b| b.buffer());- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());-- let state_buf_rc = state_array.as_ref().map(|s| s.buffer());- let state_buf_borrow = state_buf_rc.as_ref().map(|rc| rc.borrow());+ let state_array =+ (!state_in_place).then(|| context.create_array_from(&[state_allocation_size], &state_data));let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- in_proj_array.buffer().borrow().deref(),- w_array.buffer().borrow().deref(),- bias_buf_borrow.as_deref(),- state_buf_borrow.as_deref(),- out_array.buffer().borrow_mut().deref_mut(),- next_state_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.allocation(),+ w_array.allocation(),+ b_array.as_ref().map(|bias| bias.allocation()),+ state_array.as_ref().map(|state| state.allocation()),+ &mut out,+ &mut next_state,input.suffix_len,input.kernel_size,input.in_proj_stride,@@ -91,7 +79,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>();encoder.end_encoding().submit().wait_until_completed().unwrap();- (out_array.as_slice().to_vec(), next_state_array.as_slice().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&out),+ crate::common::helpers::allocation_to_vec(&next_state),+ )}fn get_test_data_basic<T: ArrayElement + Float>(diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rsindex 0bfb0e1ed..d9c935afc 100644--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_pack_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -30,18 +29,20 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(&context, T::data_type()).expect("Failed to create ShortConvPackKernel");- let state_in_array = context.create_array_from(&[input.state_in.len()], &input.state_in, "");- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");+ let state_in_array = context.create_array_from(&[input.state_in.len()], &input.state_in);+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);let padded_rows = (input.state_stride + input.suffix_len) as usize;let padded_size = padded_rows * input.model_dim as usize;- let padded_array = context.create_array_uninitialized(&[padded_size], T::data_type(), "");+ let mut padded = context+ .create_array_uninitialized(&[padded_size], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- state_in_array.buffer().borrow().deref(),- in_proj_array.buffer().borrow().deref(),- padded_array.buffer().borrow_mut().deref_mut(),+ state_in_array.allocation(),+ in_proj_array.allocation(),+ &mut padded,input.state_stride,input.suffix_len,input.in_proj_stride,@@ -50,7 +51,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- padded_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&padded)}/// Build test input for ShortConvPack.diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rsindex 45fd94d33..8723c762a 100644--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_prefill_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -34,28 +33,29 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(&context, T::data_type(), has_bias).expect("Failed to create ShortConvPrefillKernel");- let padded_array = context.create_array_from(&[input.padded.len()], &input.padded, "");- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");- let w_array = context.create_array_from(&[input.w.len()], &input.w, "");- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, ""));+ let padded_array = context.create_array_from(&[input.padded.len()], &input.padded);+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);+ let w_array = context.create_array_from(&[input.w.len()], &input.w);+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));let out_size = input.suffix_len as usize * input.model_dim as usize;- let out_array = context.create_array_uninitialized(&[out_size], T::data_type(), "");+ let mut out = context+ .create_array_uninitialized(&[out_size], T::data_type())+ .into_allocation();let state_out_size = input.model_dim as usize * input.state_stride as usize;- let state_out_array = context.create_array_uninitialized(&[state_out_size], T::data_type(), "");-- let bias_buf_rc = b_array.as_ref().map(|b| b.buffer());- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());+ let mut state_out = context+ .create_array_uninitialized(&[state_out_size], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- padded_array.buffer().borrow().deref(),- in_proj_array.buffer().borrow().deref(),- w_array.buffer().borrow().deref(),- bias_buf_borrow.as_deref(),- out_array.buffer().borrow_mut().deref_mut(),- state_out_array.buffer().borrow_mut().deref_mut(),+ padded_array.allocation(),+ in_proj_array.allocation(),+ w_array.allocation(),+ b_array.as_ref().map(|bias| bias.allocation()),+ &mut out,+ &mut state_out,input.suffix_len,input.kernel_size,input.in_proj_stride,@@ -65,7 +65,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,);encoder.end_encoding().submit().wait_until_completed().unwrap();- (out_array.as_slice().to_vec(), state_out_array.as_slice().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&out),+ crate::common::helpers::allocation_to_vec(&state_out),+ )}fn get_test_data_basic<T: ArrayElement + Float>(diff --git a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rsindex db7245033..c0a5bc778 100644--- a/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/short_conv/short_conv_trie_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -32,44 +31,48 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let context = B::Context::new().expect("Failed to create Context");let has_bias = input.b.is_some();- let kernel = <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(&context, T::data_type(), has_bias)- .expect("Failed to create ShortConvTrieKernel");+ let kernel =+ <<B as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(&context, T::data_type(), has_bias)+ .expect("Failed to create ShortConvTrieKernel");- let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj, "");- let w_array = context.create_array_from(&[input.w.len()], &input.w, "");- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, ""));+ let in_proj_array = context.create_array_from(&[input.in_proj.len()], &input.in_proj);+ let w_array = context.create_array_from(&[input.w.len()], &input.w);+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));let out_size = input.suffix_len as usize * input.model_dim as usize;- let out_array = context.create_array_uninitialized(&[out_size], T::data_type(), "");+ let mut out = context.create_array_uninitialized(&[out_size], T::data_type()).into_allocation();let state_stride = input.state_stride as usize;let model_dim = input.model_dim as usize;let suffix_len = input.suffix_len as usize;let base_state_size = model_dim * state_stride;- let base_state_array = if base_state_size > 0 {- context.create_array_from(&[base_state_size], &input.base_state, "")- } else {- context.create_array_uninitialized(&[1], T::data_type(), "")- };-- let parents_array = context.create_array_from(&[input.parents.len()], &input.parents, "");+ let base_state_allocation_size = base_state_size.max(1);+ let base_state_data: Vec<T> = input+ .base_state+ .iter()+ .copied()+ .chain(std::iter::repeat(T::zero()))+ .take(base_state_allocation_size)+ .collect();+ let base_state_array = context.create_array_from(&[base_state_allocation_size], &base_state_data);++ let parents_array = context.create_array_from(&[input.parents.len()], &input.parents);let suffix_state_size = suffix_len * model_dim * state_stride;- let suffix_state_array = context.create_array_uninitialized(&[suffix_state_size.max(1)], T::data_type(), "");-- let bias_buf_rc = b_array.as_ref().map(|b| b.buffer());- let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());+ let mut suffix_state = context+ .create_array_uninitialized(&[suffix_state_size.max(1)], T::data_type())+ .into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- in_proj_array.buffer().borrow().deref(),- w_array.buffer().borrow().deref(),- bias_buf_borrow.as_deref(),- base_state_array.buffer().borrow().deref(),- parents_array.buffer().borrow().deref(),- out_array.buffer().borrow_mut().deref_mut(),- suffix_state_array.buffer().borrow_mut().deref_mut(),+ in_proj_array.allocation(),+ w_array.allocation(),+ b_array.as_ref().map(|bias| bias.allocation()),+ base_state_array.allocation(),+ parents_array.allocation(),+ &mut out,+ &mut suffix_state,input.suffix_len,input.kernel_size,input.in_proj_stride,@@ -79,7 +82,10 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,);encoder.end_encoding().submit().wait_until_completed().unwrap();- (out_array.as_slice().to_vec(), suffix_state_array.as_slice().to_vec())+ (+ crate::common::helpers::allocation_to_vec(&out),+ crate::common::helpers::allocation_to_vec(&suffix_state),+ )}/// Linear chain: node 0 has parent -1 (root), node i has parent i-1.diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rsindex 0339d5d58..8bad6a293 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_decode_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -101,31 +100,27 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tlet c_out_size = input.suffix_len as usize * input.proj_dim as usize;let state_size = input.num_channels as usize * input.state_stride as usize;- let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");- let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));+ let x_array = context.create_array_from(&[input.x.len()], &input.x);+ let w_array = context.create_array_from(&[input.w.len()], &input.w);+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));- let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");- let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");- let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");-- let b_buf = b_array.as_ref().map(|a| a.buffer());- let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());- let b_deref: Option<&B::DenseBuffer> = b_borrow.as_ref().map(|b| b.deref());+ let mut x_out = context.create_array_uninitialized(&[x_out_size], T::data_type()).into_allocation();+ let mut b_out = context.create_array_uninitialized(&[b_out_size], T::data_type()).into_allocation();+ let mut c_out = context.create_array_uninitialized(&[c_out_size], T::data_type()).into_allocation();if input.state_in_place {- let next_state_array = context.create_array_from(&[state_size], &input.state, "next_state");+ let mut next_state = context.create_array_from(&[state_size], &input.state).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- x_array.buffer().borrow().deref(),- w_array.buffer().borrow().deref(),- b_deref,- None::<&B::DenseBuffer>,- x_out_array.buffer().borrow_mut().deref_mut(),- b_out_array.buffer().borrow_mut().deref_mut(),- c_out_array.buffer().borrow_mut().deref_mut(),- next_state_array.buffer().borrow_mut().deref_mut(),+ x_array.allocation(),+ w_array.allocation(),+ b_array.as_ref().map(|bias| bias.allocation()),+ None::<&backend_uzu::backends::common::Allocation<B>>,+ &mut x_out,+ &mut b_out,+ &mut c_out,+ &mut next_state,input.kernel_size,input.row_stride,input.state_stride,@@ -139,25 +134,26 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tencoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");Output {- x_out: x_out_array.as_slice().to_vec(),- b_out: b_out_array.as_slice().to_vec(),- c_out: c_out_array.as_slice().to_vec(),- next_state: next_state_array.as_slice().to_vec(),+ x_out: crate::common::helpers::allocation_to_vec(&x_out),+ b_out: crate::common::helpers::allocation_to_vec(&b_out),+ c_out: crate::common::helpers::allocation_to_vec(&c_out),+ next_state: crate::common::helpers::allocation_to_vec(&next_state),}} else {- let state_array = context.create_array_from(&[state_size], &input.state, "state");- let next_state_array = context.create_array_uninitialized(&[state_size], T::data_type(), "next_state");+ let state_array = context.create_array_from(&[state_size], &input.state);+ let mut next_state =+ context.create_array_uninitialized(&[state_size], T::data_type()).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- x_array.buffer().borrow().deref(),- w_array.buffer().borrow().deref(),- b_deref,- Some(state_array.buffer().borrow().deref()),- x_out_array.buffer().borrow_mut().deref_mut(),- b_out_array.buffer().borrow_mut().deref_mut(),- c_out_array.buffer().borrow_mut().deref_mut(),- next_state_array.buffer().borrow_mut().deref_mut(),+ x_array.allocation(),+ w_array.allocation(),+ b_array.as_ref().map(|bias| bias.allocation()),+ Some(state_array.allocation()),+ &mut x_out,+ &mut b_out,+ &mut c_out,+ &mut next_state,input.kernel_size,input.row_stride,input.state_stride,@@ -171,10 +167,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tencoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");Output {- x_out: x_out_array.as_slice().to_vec(),- b_out: b_out_array.as_slice().to_vec(),- c_out: c_out_array.as_slice().to_vec(),- next_state: next_state_array.as_slice().to_vec(),+ x_out: crate::common::helpers::allocation_to_vec(&x_out),+ b_out: crate::common::helpers::allocation_to_vec(&b_out),+ c_out: crate::common::helpers::allocation_to_vec(&c_out),+ next_state: crate::common::helpers::allocation_to_vec(&next_state),}}}diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rsindex 502f49e77..1e2de9263 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_pack_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -56,15 +55,15 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let total_rows = input.state_stride as usize + input.suffix_len as usize;let padded_size = total_rows * input.row_stride as usize;- let state_array = context.create_array_from(&[state_size], &input.state_in, "state_in");- let x_array = context.create_array_from(&[x_size], &input.x, "x");- let padded_array = context.create_array_uninitialized(&[padded_size], T::data_type(), "padded");+ let state_array = context.create_array_from(&[state_size], &input.state_in);+ let x_array = context.create_array_from(&[x_size], &input.x);+ let mut padded = context.create_array_uninitialized(&[padded_size], T::data_type()).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- state_array.buffer().borrow().deref(),- x_array.buffer().borrow().deref(),- padded_array.buffer().borrow_mut().deref_mut(),+ state_array.allocation(),+ x_array.allocation(),+ &mut padded,input.state_stride,input.row_stride,input.suffix_len,@@ -73,7 +72,7 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");- padded_array.as_slice().to_vec()+ crate::common::helpers::allocation_to_vec(&padded)}fn get_test_data<T: ArrayElement + Float>(diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rsindex 99ccd937c..6be3cc6b2 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_conv1d_scan_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -90,28 +89,25 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tlet c_out_size = input.suffix_len as usize * input.proj_dim as usize;let state_size = input.num_channels as usize * input.state_stride as usize;- let padded_array = context.create_array_from(&[input.padded.len()], &input.padded, "padded");- let w_array = context.create_array_from(&[input.w.len()], &input.w, "w");- let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b, "b"));+ let padded_array = context.create_array_from(&[input.padded.len()], &input.padded);+ let w_array = context.create_array_from(&[input.w.len()], &input.w);+ let b_array = input.b.as_ref().map(|b| context.create_array_from(&[b.len()], b));- let x_out_array = context.create_array_uninitialized(&[x_out_size], T::data_type(), "x_out");- let b_out_array = context.create_array_uninitialized(&[b_out_size], T::data_type(), "b_out");- let c_out_array = context.create_array_uninitialized(&[c_out_size], T::data_type(), "c_out");- let state_out_array = context.create_array_uninitialized(&[state_size], T::data_type(), "state_out");-- let b_buf = b_array.as_ref().map(|a| a.buffer());- let b_borrow = b_buf.as_ref().map(|rc| rc.borrow());- let b_deref: Option<&B::DenseBuffer> = b_borrow.as_ref().map(|b| b.deref());+ let mut x_out = context.create_array_uninitialized(&[x_out_size], T::data_type()).into_allocation();+ let mut b_out = context.create_array_uninitialized(&[b_out_size], T::data_type()).into_allocation();+ let mut c_out = context.create_array_uninitialized(&[c_out_size], T::data_type()).into_allocation();+ let mut state_out =+ context.create_array_uninitialized(&[state_size], T::data_type()).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- padded_array.buffer().borrow().deref(),- w_array.buffer().borrow().deref(),- b_deref,- x_out_array.buffer().borrow_mut().deref_mut(),- b_out_array.buffer().borrow_mut().deref_mut(),- c_out_array.buffer().borrow_mut().deref_mut(),- state_out_array.buffer().borrow_mut().deref_mut(),+ padded_array.allocation(),+ w_array.allocation(),+ b_array.as_ref().map(|bias| bias.allocation()),+ &mut x_out,+ &mut b_out,+ &mut c_out,+ &mut state_out,input.suffix_len,input.kernel_size,input.row_stride,@@ -125,10 +121,10 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tencoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");Output {- x_out: x_out_array.as_slice().to_vec(),- b_out: b_out_array.as_slice().to_vec(),- c_out: c_out_array.as_slice().to_vec(),- state_out: state_out_array.as_slice().to_vec(),+ x_out: crate::common::helpers::allocation_to_vec(&x_out),+ b_out: crate::common::helpers::allocation_to_vec(&b_out),+ c_out: crate::common::helpers::allocation_to_vec(&c_out),+ state_out: crate::common::helpers::allocation_to_vec(&state_out),}}diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rsindex c2bc85e19..0269536f6 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_prefill_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -102,14 +101,14 @@ fn get_output<B: Backend, T: ArrayElement + Float>(let total_x = input.suffix_len * input.num_heads * input.head_dim;- let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");- let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");- let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");- let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");- let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");- let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");- let state_array = context.create_array_from(&[input.state.len()], &input.state, "state");- let y_array = context.create_array_uninitialized(&[total_x], T::data_type(), "y");+ let x_array = context.create_array_from(&[input.x.len()], &input.x);+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt);+ let b_array = context.create_array_from(&[input.b.len()], &input.b);+ let c_array = context.create_array_from(&[input.c.len()], &input.c);+ let d_array = context.create_array_from(&[input.d.len()], &input.d);+ let z_array = context.create_array_from(&[input.z.len()], &input.z);+ let mut state = context.create_array_from(&[input.state.len()], &input.state).into_allocation();+ let mut y = context.create_array_uninitialized(&[total_x], T::data_type()).into_allocation();let x_strides: [u32; 3] = input.x_strides.map(|s| s as u32);let dt_strides: [u32; 2] = input.dt_strides.map(|s| s as u32);@@ -122,14 +121,14 @@ fn get_output<B: Backend, T: ArrayElement + Float>(let kernel = <<B as Backend>::Kernels as Kernels>::SSDPrefillKernel::new(&context, T::data_type()).expect("Failed to create SSDPrefillKernel");kernel.encode(- x_array.buffer().borrow().deref(),- dt_array.buffer().borrow().deref(),- b_array.buffer().borrow().deref(),- c_array.buffer().borrow().deref(),- d_array.buffer().borrow().deref(),- z_array.buffer().borrow().deref(),- state_array.buffer().borrow_mut().deref_mut(),- y_array.buffer().borrow_mut().deref_mut(),+ x_array.allocation(),+ dt_array.allocation(),+ b_array.allocation(),+ c_array.allocation(),+ d_array.allocation(),+ z_array.allocation(),+ &mut state,+ &mut y,input.suffix_len as u32,input.group_size as u32,input.state_dim as u32,@@ -147,14 +146,14 @@ fn get_output<B: Backend, T: ArrayElement + Float>(<<B as Backend>::Kernels as Kernels>::SSDPrefillSequentialKernel::new(&context, T::data_type()).expect("Failed to create SSDPrefillSequentialKernel");kernel.encode(- x_array.buffer().borrow().deref(),- dt_array.buffer().borrow().deref(),- b_array.buffer().borrow().deref(),- c_array.buffer().borrow().deref(),- d_array.buffer().borrow().deref(),- z_array.buffer().borrow().deref(),- state_array.buffer().borrow_mut().deref_mut(),- y_array.buffer().borrow_mut().deref_mut(),+ x_array.allocation(),+ dt_array.allocation(),+ b_array.allocation(),+ c_array.allocation(),+ d_array.allocation(),+ z_array.allocation(),+ &mut state,+ &mut y,input.suffix_len as u32,input.group_size as u32,input.state_dim as u32,@@ -171,8 +170,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");Output {- y: y_array.as_slice().to_vec(),- state: state_array.as_slice().to_vec(),+ y: crate::common::helpers::allocation_to_vec(&y),+ state: crate::common::helpers::allocation_to_vec(&state),}}diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rsindex b443d33c0..0d15b3a64 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_split_inproj_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -63,20 +62,21 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tlet z_out_size = input.suffix_length as usize * input.inner_dim as usize;let dt_out_size = input.suffix_length as usize * input.num_heads as usize;- let input_array = context.create_array_from(&[input.input.len()], &input.input, "input");- let z_bias_array = context.create_array_from(&[input.z_bias.len()], &input.z_bias, "z_bias");+ let input_array = context.create_array_from(&[input.input.len()], &input.input);+ let z_bias_array = context.create_array_from(&[input.z_bias.len()], &input.z_bias);- let conv_out_array = context.create_array_uninitialized(&[conv_out_size], T::data_type(), "conv_out");- let z_out_array = context.create_array_uninitialized(&[z_out_size], T::data_type(), "z_out");- let dt_out_array = context.create_array_uninitialized(&[dt_out_size], T::data_type(), "dt_out");+ let mut conv_out =+ context.create_array_uninitialized(&[conv_out_size], T::data_type()).into_allocation();+ let mut z_out = context.create_array_uninitialized(&[z_out_size], T::data_type()).into_allocation();+ let mut dt_out = context.create_array_uninitialized(&[dt_out_size], T::data_type()).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- input_array.buffer().borrow().deref(),- conv_out_array.buffer().borrow_mut().deref_mut(),- z_out_array.buffer().borrow_mut().deref_mut(),- dt_out_array.buffer().borrow_mut().deref_mut(),- z_bias_array.buffer().borrow().deref(),+ input_array.allocation(),+ &mut conv_out,+ &mut z_out,+ &mut dt_out,+ z_bias_array.allocation(),input.suffix_length,input.total_dim,input.conv_dim,@@ -87,9 +87,9 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tencoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");Output {- conv_out: conv_out_array.as_slice().to_vec(),- z_out: z_out_array.as_slice().to_vec(),- dt_out: dt_out_array.as_slice().to_vec(),+ conv_out: crate::common::helpers::allocation_to_vec(&conv_out),+ z_out: crate::common::helpers::allocation_to_vec(&z_out),+ dt_out: crate::common::helpers::allocation_to_vec(&dt_out),}}diff --git a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rsindex 35b4bc072..a71cc9955 100644--- a/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/ssm/ssm_update_test.rs@@ -1,6 +1,5 @@use std::{fmt::{Debug, Display},- ops::{Deref, DerefMut},};use backend_uzu::{@@ -96,29 +95,29 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tlet y_size = bsz * h * dh;let ns_size = bsz * h * dh * n;- let x_array = context.create_array_from(&[input.x.len()], &input.x, "x");- let dt_array = context.create_array_from(&[input.dt.len()], &input.dt, "dt");- let b_array = context.create_array_from(&[input.b.len()], &input.b, "b");- let c_array = context.create_array_from(&[input.c.len()], &input.c, "c");- let d_array = context.create_array_from(&[input.d.len()], &input.d, "d");- let z_array = context.create_array_from(&[input.z.len()], &input.z, "z");+ let x_array = context.create_array_from(&[input.x.len()], &input.x);+ let dt_array = context.create_array_from(&[input.dt.len()], &input.dt);+ let b_array = context.create_array_from(&[input.b.len()], &input.b);+ let c_array = context.create_array_from(&[input.c.len()], &input.c);+ let d_array = context.create_array_from(&[input.d.len()], &input.d);+ let z_array = context.create_array_from(&[input.z.len()], &input.z);- let y_array = context.create_array_uninitialized(&[y_size], T::data_type(), "y");+ let mut y = context.create_array_uninitialized(&[y_size], T::data_type()).into_allocation();if input.state_in_place {- let ns_array = context.create_array_from(&[ns_size], &input.state, "next_state");+ let mut next_state = context.create_array_from(&[ns_size], &input.state).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- x_array.buffer().borrow().deref(),- dt_array.buffer().borrow().deref(),- b_array.buffer().borrow().deref(),- c_array.buffer().borrow().deref(),- d_array.buffer().borrow().deref(),- z_array.buffer().borrow().deref(),- None::<&<B as Backend>::DenseBuffer>,- y_array.buffer().borrow_mut().deref_mut(),- ns_array.buffer().borrow_mut().deref_mut(),+ x_array.allocation(),+ dt_array.allocation(),+ b_array.allocation(),+ c_array.allocation(),+ d_array.allocation(),+ z_array.allocation(),+ None::<&backend_uzu::backends::common::Allocation<B>>,+ &mut y,+ &mut next_state,(h / g) as u32,input.n,&x_strides,@@ -133,24 +132,25 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tencoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");Output {- y: y_array.as_slice().to_vec(),- next_state: ns_array.as_slice().to_vec(),+ y: crate::common::helpers::allocation_to_vec(&y),+ next_state: crate::common::helpers::allocation_to_vec(&next_state),}} else {- let state_array = context.create_array_from(&[ns_size], &input.state, "state");- let ns_array = context.create_array_uninitialized(&[ns_size], T::data_type(), "next_state");+ let state_array = context.create_array_from(&[ns_size], &input.state);+ let mut next_state =+ context.create_array_uninitialized(&[ns_size], T::data_type()).into_allocation();let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- x_array.buffer().borrow().deref(),- dt_array.buffer().borrow().deref(),- b_array.buffer().borrow().deref(),- c_array.buffer().borrow().deref(),- d_array.buffer().borrow().deref(),- z_array.buffer().borrow().deref(),- Some(state_array.buffer().borrow().deref()),- y_array.buffer().borrow_mut().deref_mut(),- ns_array.buffer().borrow_mut().deref_mut(),+ x_array.allocation(),+ dt_array.allocation(),+ b_array.allocation(),+ c_array.allocation(),+ d_array.allocation(),+ z_array.allocation(),+ Some(state_array.allocation()),+ &mut y,+ &mut next_state,(h / g) as u32,input.n,&x_strides,@@ -165,8 +165,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Output<Tencoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");Output {- y: y_array.as_slice().to_vec(),- next_state: ns_array.as_slice().to_vec(),+ y: crate::common::helpers::allocation_to_vec(&y),+ next_state: crate::common::helpers::allocation_to_vec(&next_state),}}}diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rsindex 6081d5817..7a499c6ab 100644--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_bias_test.rs@@ -1,16 +1,16 @@-use std::{- fmt::Debug,- ops::{Deref, DerefMut},-};+use std::fmt::Debug;+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{- ArrayContextExt, ArrayElement,+ ArrayElement,backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},};-use half::{bf16, f16};-use num_traits::Float;-use crate::uzu_test;+use crate::{+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ uzu_test,+};struct Input<T: ArrayElement + Float> {input: Box<[T]>,@@ -64,30 +64,25 @@ fn get_output<T: ArrayElement + Float, B: Backend>(.expect("Failed to create TensorAddBiasKernel");let size = input.length as usize;- let input_array = context.create_array_from(&[size], &input.input, "");- let input_array_buffer_rc = input_array.buffer();- let input_array_borrow = input_array_buffer_rc.borrow();- let input_array_deref = input_array_borrow.deref();- let input_buffer = (!in_place).then(|| input_array_deref);-- let bias_array = context.create_array_from(&[input.num_cols as usize], &input.bias, "");- let output_array = match in_place {- true => context.create_array_from(&[size], &input.output, ""),- false => context.create_array_uninitialized(&[size], T::data_type(), ""),+ let input_allocation = (!in_place).then(|| alloc_allocation_with_data::<B, T>(&context, &input.input));+ let bias_allocation = alloc_allocation_with_data::<B, T>(&context, &input.bias);+ let mut output_allocation = match in_place {+ true => alloc_allocation_with_data::<B, T>(&context, &input.output),+ false => alloc_allocation::<B, T>(&context, size),};let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- input_buffer,- bias_array.buffer().borrow_mut().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ input_allocation.as_ref(),+ &bias_allocation,+ &mut output_allocation,input.num_cols,input.length,&mut encoder,);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&output_allocation)}fn test<T: ArrayElement + Float + Debug>(in_place: bool) {diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rsindex 50d516aea..4355a7f56 100644--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_scale_test.rs@@ -1,19 +1,22 @@-use std::{- fmt::{Debug, Display},- ops::{Deref, DerefMut},-};+use std::fmt::{Debug, Display};+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{- ArrayContextExt, ArrayElement, DataType,+ ArrayElement, DataType,backends::{common::{Backend, Context, Encoder, Kernels, kernel::TensorAddScaleKernel},cpu::Cpu,},};-use half::{bf16, f16};-use num_traits::Float;-use crate::{common::assert::assert_eq_float, uzu_test};+use crate::{+ common::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+ uzu_test,+};struct Input<T: ArrayElement + Float> {input: Box<[T]>,@@ -41,15 +44,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let length = input.length as usize;let num_cols = input.num_cols as usize;- let input_array = context.create_array_from(&[length], &input.input, "");- let bias_array = context.create_array_from(&[num_cols], &input.bias, "");- let output_array = context.create_array_uninitialized(&[length], T::data_type(), "");+ let input_allocation = alloc_allocation_with_data::<B, T>(&context, &input.input[..length]);+ let bias_allocation = alloc_allocation_with_data::<B, T>(&context, &input.bias[..num_cols]);+ let mut output_allocation = alloc_allocation::<B, T>(&context, length);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(- input_array.buffer().borrow().deref(),- bias_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),+ &input_allocation,+ &bias_allocation,+ &mut output_allocation,input.num_cols,input.length,input.scale,@@ -57,7 +60,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {);encoder.end_encoding().submit().wait_until_completed().unwrap();- output_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&output_allocation)}fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rsindex 6c99b5722..69b25307d 100644--- a/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/tensor_add_swap_test.rs@@ -1,13 +1,16 @@-use std::{fmt::Debug, ops::DerefMut};+use std::fmt::Debug;+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{- ArrayContextExt, ArrayElement,+ ArrayElement,backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddSwapKernel},};-use half::{bf16, f16};-use num_traits::Float;-use crate::uzu_test;+use crate::{+ common::helpers::{alloc_allocation_with_data, allocation_to_vec},+ uzu_test,+};struct Input<T: ArrayElement + Float> {skip_buffer: Box<[T]>,@@ -44,19 +47,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,.expect("Failed to create TensorAddSwapKernel");let size = input.length as usize;- let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");- let main_array = context.create_array_from(&[size], &input.main_buffer, "");+ let mut skip_allocation = alloc_allocation_with_data::<B, T>(&context, &input.skip_buffer[..size]);+ let mut main_allocation = alloc_allocation_with_data::<B, T>(&context, &input.main_buffer[..size]);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- skip_array.buffer().borrow_mut().deref_mut(),- main_array.buffer().borrow_mut().deref_mut(),- input.length,- &mut encoder,- );+ kernel.encode(&mut skip_allocation, &mut main_allocation, input.length, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();- (skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())+ (+ allocation_to_vec::<B, T>(&skip_allocation),+ allocation_to_vec::<B, T>(&main_allocation),+ )}fn test<T: ArrayElement + Float + Debug>() {diff --git a/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rs b/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rsindex 4adc00398..a3653860d 100644--- a/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/tensor_copy_test.rs@@ -1,16 +1,16 @@-use std::{- fmt::Debug,- ops::{Deref, DerefMut},-};+use std::fmt::Debug;+use half::{bf16, f16};+use num_traits::Float;use backend_uzu::{- ArrayContextExt, ArrayElement,+ ArrayElement,backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorCopyKernel},};-use half::{bf16, f16};-use num_traits::Float;-use crate::uzu_test;+use crate::{+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ uzu_test,+};struct Input<T: ArrayElement + Float> {src: Box<[T]>,@@ -43,19 +43,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {.expect("Failed to create TensorCopyKernel");let size = input.length as usize;- let src_array = context.create_array_from(&[size], &input.src, "");- let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");+ let src_allocation = alloc_allocation_with_data::<B, T>(&context, &input.src[..size]);+ let mut dst_allocation = alloc_allocation::<B, T>(&context, size);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- src_array.buffer().borrow().deref(),- dst_array.buffer().borrow_mut().deref_mut(),- input.length,- &mut encoder,- );+ kernel.encode(&src_allocation, &mut dst_allocation, input.length, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();- dst_array.as_slice().to_vec()+ allocation_to_vec::<B, T>(&dst_allocation)}fn test<T: ArrayElement + Float + Debug>() {diff --git a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rsindex 565d45d53..6e02a485c 100644--- a/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/token_copy_test.rs@@ -1,14 +1,14 @@-use std::ops::{Deref, DerefMut};-use backend_uzu::{- ArrayContextExt, ArrayElement,backends::common::{Backend, Context, Encoder, Kernels,kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},},};-use crate::uzu_test;+use crate::{+ common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ uzu_test,+};fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {let context = B::Context::new().expect("Failed to create Context");@@ -16,14 +16,14 @@ fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopySampledKernel::new(&context).expect("Failed to create TokenCopySampledKernel");- let src_array = context.create_array_from(&[1], &[src_value], "");- let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");+ let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);+ let mut dst_allocation = alloc_allocation::<B, u64>(&context, 1);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);+ kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();- let output = dst_array.as_slice::<u64>().to_vec();+ let output = allocation_to_vec::<B, u64>(&dst_allocation);assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());}@@ -33,14 +33,14 @@ fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {let kernel = <<B as Backend>::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context).expect("Failed to create TokenCopyToResultsKernel");- let src_array = context.create_array_from(&[1], &[src_value], "");- let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");+ let src_allocation = alloc_allocation_with_data::<B, u32>(&context, &[src_value]);+ let mut dst_allocation = alloc_allocation::<B, u32>(&context, 1);let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);+ kernel.encode(&src_allocation, &mut dst_allocation, &mut encoder);encoder.end_encoding().submit().wait_until_completed().unwrap();- let output = dst_array.as_slice::<u32>().to_vec();+ let output = allocation_to_vec::<B, u32>(&dst_allocation);assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());}