A high-performance inference engine for AI models
new encoder (#236)
uuuvn5cfcf873d7d6
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rsindex ffb269a5d..6b0e8fd4a 100644--- a/crates/uzu/build/common/traitgen.rs+++ b/crates/uzu/build/common/traitgen.rs@@ -19,7 +19,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {let params = kernel.parameters.iter().map(|p| {let name = format_ident!("{}", p.name.as_ref());let ty = match &p.ty {- KernelParameterType::Type => quote! { DataType },+ KernelParameterType::Type => quote! { crate::DataType },KernelParameterType::Value(ty) => {let ty: Type = syn::parse_str(ty.as_ref()).unwrap();quote! { #ty }@@ -42,10 +42,10 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {Some(quote! { #buffer_lifetime }),match access {KernelBufferAccess::Read => {- quote! { impl BufferArg<#buffer_lifetime, <Self::Backend as Backend>::NativeBuffer> }+ quote! { impl BufferArg<#buffer_lifetime, <Self::Backend as crate::backends::common::Backend>::Buffer> }},KernelBufferAccess::ReadWrite => {- quote! { impl BufferArgMut<#buffer_lifetime, <Self::Backend as Backend>::NativeBuffer> }+ quote! { impl BufferArgMut<#buffer_lifetime, <Self::Backend as crate::backends::common::Backend>::Buffer> }},},)@@ -72,12 +72,11 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {let kernel_trait = quote! {pub trait #trait_name: Sized {- type Backend: Backend<Kernels: Kernels<#trait_name = Self>>;+ type Backend: crate::backends::common::Backend<Kernels: Kernels<#trait_name = Self>>;- fn new(context: &<Self::Backend as Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as Backend>::Error>;+ fn new(context: &<Self::Backend as crate::backends::common::Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as crate::backends::common::Backend>::Error>;- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder);- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder, predicate: Option<impl BufferArg<'predicate, <Self::Backend as Backend>::NativeBuffer>>);+ fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#args ,)* command_buffer: &'command_buffer mut <<Self::Backend as crate::backends::common::Backend>::CommandBuffer as crate::backends::common::CommandBuffer>::Encoding);}};@@ -114,36 +113,33 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>}let kernel_traits = quote! {- use crate::backends::common::{Backend, NativeBuffer};- use crate::DataType;-- pub trait BufferArg<'a, B: NativeBuffer> {+ pub trait BufferArg<'a, B: crate::backends::common::Buffer> {fn into_parts(self) -> (&'a B, usize);}- impl<'a, B: NativeBuffer> BufferArg<'a, B> for &'a B {+ impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for &'a B {fn into_parts(self) -> (&'a B, usize) {(self, 0)}}- impl<'a, B: NativeBuffer> BufferArg<'a, B> for (&'a B, usize) {+ impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for (&'a B, usize) {fn into_parts(self) -> (&'a B, usize) {self}}- pub trait BufferArgMut<'a, B: NativeBuffer> {+ pub trait BufferArgMut<'a, B: crate::backends::common::Buffer> {fn into_parts(self) -> (&'a mut B, usize);}- impl<'a, B: NativeBuffer> BufferArgMut<'a, B> for &'a mut B {+ impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for &'a mut B {fn into_parts(self) -> (&'a mut B, usize) {(self, 0)}}- impl<'a, B: NativeBuffer> BufferArgMut<'a, B> for (&'a mut B, usize) {+ impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for (&'a mut B, usize) {fn into_parts(self) -> (&'a mut B, usize) {self}@@ -152,7 +148,7 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>#(#kernel_traits)*pub trait Kernels: Sized {- type Backend: Backend<Kernels = Self>;+ type Backend: crate::backends::common::Backend<Kernels = Self>;#(#kernel_types)*}diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rsindex 971d14281..da8c99615 100644--- a/crates/uzu/build/cpu/compiler.rs+++ b/crates/uzu/build/cpu/compiler.rs@@ -329,7 +329,7 @@ impl CpuCompiler {}).collect::<anyhow::Result<_>>()?;- let (encode_generics, encode_args_defs, encode_args_passthrough): (Vec<_>, Vec<_>, Vec<_>) = kernel_arguments+ let (encode_generics, encode_args_defs): (Vec<_>, Vec<_>) = kernel_arguments.iter().map(|argument| {let argument_ident: Ident = syn::parse_str(argument.name.as_ref()).context("cannot parse ident")?;@@ -359,7 +359,7 @@ impl CpuCompiler {ty = quote! { Option<#ty> };}- Ok((generic, quote! { #argument_ident: #ty }, quote! { #argument_ident }))+ Ok((generic, quote! { #argument_ident: #ty }))}).collect::<anyhow::Result<_>>()?;@@ -439,12 +439,7 @@ impl CpuCompiler {});quote! {- encoder.push_command(move || {- if let Some(predicate) = predicate && unsafe { *predicate } == 0u32 {- return;- }- #monomorphized_function(#function_call_args_joined)- });+ encoder.push_command(move || #monomorphized_function(#function_call_args_joined));}};@@ -536,16 +531,7 @@ impl CpuCompiler {}fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer) {- self.encode_if(#(#encode_args_passthrough ,)* encoder, None::<&Box<[u8]>>)- }-- fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {#(#argument_copies)*- let predicate = predicate.map(|__dsl_buffer_impl| unsafe {- let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();-- __dsl_buffer.as_ptr().byte_add(__dsl_offset) as *const u32- });#encode_body}}diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex 0db3014f3..309c6bb7a 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -99,13 +99,12 @@ pub fn bindgen(let mut arg_count: usize = 0;let mut indirect_flag = false;- let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_args_sets, encode_args_names): (+ let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_args_sets): (Vec<Option<TokenStream>>,Vec<Option<TokenStream>>,Vec<Option<TokenStream>>,Vec<TokenStream>,Vec<TokenStream>,- Vec<TokenStream>,) = kernel.arguments.iter()@@ -176,13 +175,12 @@ pub fn bindgen(generic,quote! { #arg_name: #ty },set,- quote! { #arg_name },))}MetalArgumentType::Groups(MetalGroupsType::Indirect) if !indirect_flag => {indirect_flag = true;- Some((None, None, Some(quote! { '__dsl_indirect_dispatch_buffer }), quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> }, quote! {}, quote! { __dsl_indirect_dispatch_buffer }))+ Some((None, None, Some(quote! { '__dsl_indirect_dispatch_buffer }), quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> }, quote! {}))}_ => None,}@@ -323,29 +321,13 @@ pub fn bindgen(Ok(Self { pipeline #(, #conditional_buffer_sets)* })}- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {+ fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {+ let compute_encoder = command_buffer.ensure_compute();#empty_dispatch_guardscompute_encoder.set_compute_pipeline_state(&self.pipeline);#(#encode_args_sets)*#dispatch}- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {- #empty_dispatch_guards-- if let Some(predicate) = predicate {- let (__dsl_buffer, __dsl_offset) = predicate.into_parts();- compute_encoder.condition(- __dsl_buffer,- __dsl_offset,- |compute_encoder| {- self.encode(#(#encode_args_names, )* compute_encoder);- },- None::<fn(&mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>)>,- );- } else {- self.encode(#(#encode_args_names, )* compute_encoder);- }- }}};diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rsindex 1120c6848..ef53773b3 100644--- a/crates/uzu/build/metal/compiler.rs+++ b/crates/uzu/build/metal/compiler.rs@@ -288,9 +288,10 @@ impl MetalCompiler {use crate::backends::metal::{context::MetalContext,+ command_buffer::MetalCommandBufferEncoding,error::MetalError,metal_extensions::{- ComputeEncoderConditional, ComputeEncoderSetValue, FunctionConstantValuesSetValue, MetalDataTypeExt,+ ComputeEncoderSetValue, FunctionConstantValuesSetValue, MetalDataTypeExt,},};diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rsindex 20b7f16e1..2c3c3fe9b 100644--- a/crates/uzu/src/array.rs+++ b/crates/uzu/src/array.rs@@ -4,12 +4,12 @@ use ndarray::{ArrayView, Dimension, IxDyn};use crate::{ArrayElement, DataType,- backends::common::{Backend, Context, NativeBuffer},+ backends::common::{Backend, Buffer, Context},};#[derive(Debug)]pub struct Array<B: Backend> {- buffer: Rc<RefCell<B::NativeBuffer>>,+ buffer: Rc<RefCell<B::Buffer>>,offset: usize,shape: Box<[usize]>,data_type: DataType,@@ -18,7 +18,7 @@ pub struct Array<B: Backend> {impl<B: Backend> Array<B> {// Constructorspub unsafe fn from_parts(- buffer: Rc<RefCell<B::NativeBuffer>>,+ buffer: Rc<RefCell<B::Buffer>>,offset: usize,shape: &[usize],data_type: DataType,@@ -49,7 +49,7 @@ impl<B: Backend> Array<B> {}// Getters- pub fn buffer(&self) -> Rc<RefCell<B::NativeBuffer>> {+ pub fn buffer(&self) -> Rc<RefCell<B::Buffer>> {self.buffer.clone()}diff --git a/crates/uzu/src/backends/common/allocator/allocator.rs b/crates/uzu/src/backends/common/allocator/allocator.rsdeleted file mode 100644index 66bf849b1..000000000--- a/crates/uzu/src/backends/common/allocator/allocator.rs+++ /dev/null@@ -1,186 +0,0 @@-use std::{- cell::{Cell, RefCell},- rc::Weak,-};--use super::{- AllocError,- scratch_pool::{ScratchPool, align_size},-};-use crate::backends::common::{Backend, Buffer, BufferLifetime, Context, NativeBuffer};--pub struct Allocator<B: Backend> {- context: Weak<B::Context>,- scratch_pool: RefCell<ScratchPool<B::NativeBuffer>>,- active_memory: Cell<usize>,- peak_memory: Cell<usize>,- gc_limit: Cell<usize>,- max_pool_size: Cell<usize>,-}--impl<B: Backend> Allocator<B> {- pub fn new(context: Weak<B::Context>) -> Self {- Self {- context,- scratch_pool: RefCell::new(ScratchPool::new()),- active_memory: Cell::new(0),- peak_memory: Cell::new(0),- gc_limit: Cell::new(usize::MAX),- max_pool_size: Cell::new(usize::MAX),- }- }-- fn create_native_buffer(- &self,- size: usize,- ) -> Result<B::NativeBuffer, AllocError> {- let context = self.context.upgrade().ok_or_else(|| AllocError::AllocationFailed {- size,- reason: "context has been dropped".to_string(),- })?;- context.create_buffer(size).map_err(|e| AllocError::AllocationFailed {- size,- reason: e.to_string(),- })- }-- fn wrap_buffer(- &self,- native: B::NativeBuffer,- is_scratch: bool,- ) -> Buffer<B> {- Buffer::new(native, self.context.clone(), is_scratch)- }-- pub fn set_gc_limit(- &self,- limit: usize,- ) {- self.gc_limit.set(limit);- }-- pub fn set_max_pool_size(- &self,- limit: usize,- ) {- self.max_pool_size.set(limit);- }-- pub fn gc_limit(&self) -> usize {- self.gc_limit.get()- }-- pub fn max_pool_size(&self) -> usize {- self.max_pool_size.get()- }-- #[inline]- fn track_allocation(- &self,- size: usize,- ) {- let new_active = self.active_memory.get() + size;- self.active_memory.set(new_active);- if new_active > self.peak_memory.get() {- self.peak_memory.set(new_active);- }- }-- #[inline]- fn track_deallocation(- &self,- size: usize,- ) {- self.active_memory.set(self.active_memory.get().saturating_sub(size));- }-- fn alloc_scratch_internal(- &self,- aligned_size: usize,- ) -> Result<B::NativeBuffer, AllocError> {- let mut pool = self.scratch_pool.borrow_mut();-- if let Some((buffer, actual_size)) = pool.find_buffer(aligned_size) {- self.track_allocation(actual_size);- return Ok(buffer);- }-- let gc_limit = self.gc_limit.get();- let active = self.active_memory.get();- let cache = pool.available_size();- let mem_required = active + cache + aligned_size;-- if mem_required >= gc_limit {- let to_free = mem_required.saturating_sub(gc_limit);- pool.release_cached_buffers(to_free);- }-- pool.total_allocated += aligned_size;- drop(pool);-- let buffer = self.create_native_buffer(aligned_size)?;- self.track_allocation(aligned_size);-- Ok(buffer)- }-- pub fn alloc(- &self,- lifetime: BufferLifetime,- size: usize,- ) -> Result<Buffer<B>, AllocError> {- let aligned_size = align_size(size);-- let (native, is_scratch) = match lifetime {- BufferLifetime::Permanent => {- let buffer = self.create_native_buffer(aligned_size)?;- self.track_allocation(aligned_size);- (buffer, false)- },- BufferLifetime::Scratch => (self.alloc_scratch_internal(aligned_size)?, true),- };-- Ok(self.wrap_buffer(native, is_scratch))- }-- pub fn handle_buffer_drop(- &self,- buffer: B::NativeBuffer,- is_scratch: bool,- ) {- let size = buffer.length();- self.track_deallocation(size);-- if is_scratch {- let mut pool = self.scratch_pool.borrow_mut();- pool.return_buffer(buffer, size);-- let max_pool_size = self.max_pool_size.get();- let current_size = pool.available_size();- if current_size > max_pool_size {- let to_free = current_size - max_pool_size;- pool.release_cached_buffers(to_free);- }- }- }-- pub fn active_memory(&self) -> usize {- self.active_memory.get()- }-- pub fn peak_memory(&self) -> usize {- self.peak_memory.get()- }-- pub fn reset_peak_memory(&self) {- self.peak_memory.set(self.active_memory.get());- }-- pub fn cache_memory(&self) -> usize {- self.scratch_pool.borrow().available_size()- }-- pub fn clear_cache(&self) {- self.scratch_pool.borrow_mut().clear();- }-}diff --git a/crates/uzu/src/backends/common/allocator/error.rs b/crates/uzu/src/backends/common/allocator/error.rsdeleted file mode 100644index bb109b172..000000000--- a/crates/uzu/src/backends/common/allocator/error.rs+++ /dev/null@@ -1,37 +0,0 @@-use std::fmt;--#[derive(Debug, Clone)]-pub enum AllocError {- OutOfMemory {- requested: usize,- available: usize,- },- AllocationFailed {- size: usize,- reason: String,- },-}--impl fmt::Display for AllocError {- fn fmt(- &self,- f: &mut fmt::Formatter<'_>,- ) -> fmt::Result {- match self {- AllocError::OutOfMemory {- requested,- available,- } => {- write!(f, "Out of memory: requested {} bytes, {} available", requested, available)- },- AllocError::AllocationFailed {- size,- reason,- } => {- write!(f, "Failed to allocate {} bytes: {}", size, reason)- },- }- }-}--impl std::error::Error for AllocError {}diff --git a/crates/uzu/src/backends/common/allocator/mod.rs b/crates/uzu/src/backends/common/allocator/mod.rsdeleted file mode 100644index a7ffa15d5..000000000--- a/crates/uzu/src/backends/common/allocator/mod.rs+++ /dev/null@@ -1,6 +0,0 @@-mod allocator;-mod error;-mod scratch_pool;--pub use allocator::Allocator;-pub use error::AllocError;diff --git a/crates/uzu/src/backends/common/allocator/scratch_pool.rs b/crates/uzu/src/backends/common/allocator/scratch_pool.rsdeleted file mode 100644index 963deb3d2..000000000--- a/crates/uzu/src/backends/common/allocator/scratch_pool.rs+++ /dev/null@@ -1,255 +0,0 @@-use std::collections::BTreeMap;--pub(super) const PAGE_SIZE: usize = 16384;--pub(super) fn align_size(size: usize) -> usize {- if size > PAGE_SIZE {- (size + PAGE_SIZE - 1) & !(PAGE_SIZE - 1)- } else {- size- }-}--struct BufferNode<B> {- buffer: B,- size: usize,- bucket_idx: usize,- prev: Option<usize>,- next: Option<usize>,-}--pub(super) struct ScratchPool<B> {- nodes: Vec<Option<BufferNode<B>>>,- free_slots: Vec<usize>,- size_index: BTreeMap<usize, Vec<usize>>,- lru_head: Option<usize>,- lru_tail: Option<usize>,- pool_size: usize,- pub(super) total_allocated: usize,-}--impl<B> ScratchPool<B> {- pub(super) fn new() -> Self {- Self {- nodes: Vec::new(),- free_slots: Vec::new(),- size_index: BTreeMap::new(),- lru_head: None,- lru_tail: None,- pool_size: 0,- total_allocated: 0,- }- }-- #[inline]- fn allocate_slot(&mut self) -> usize {- self.free_slots.pop().unwrap_or_else(|| {- let slot = self.nodes.len();- self.nodes.push(None);- slot- })- }-- #[inline]- fn add_to_lru_head(- &mut self,- slot: usize,- ) {- if let Some(old_head) = self.lru_head {- if let Some(node) = self.nodes[old_head].as_mut() {- node.prev = Some(slot);- }- if let Some(node) = self.nodes[slot].as_mut() {- node.next = Some(old_head);- }- }-- self.lru_head = Some(slot);-- if self.lru_tail.is_none() {- self.lru_tail = Some(slot);- }- }-- #[inline]- fn remove_from_lru(- &mut self,- slot: usize,- ) {- let (prev, next) = match self.nodes[slot].as_ref() {- Some(node) => (node.prev, node.next),- None => return,- };-- match (prev, next) {- (Some(p), Some(n)) => {- if let Some(prev_node) = self.nodes[p].as_mut() {- prev_node.next = Some(n);- }- if let Some(next_node) = self.nodes[n].as_mut() {- next_node.prev = Some(p);- }- },- (Some(p), None) => {- if let Some(prev_node) = self.nodes[p].as_mut() {- prev_node.next = None;- }- self.lru_tail = Some(p);- },- (None, Some(n)) => {- if let Some(next_node) = self.nodes[n].as_mut() {- next_node.prev = None;- }- self.lru_head = Some(n);- },- (None, None) => {- self.lru_head = None;- self.lru_tail = None;- },- }-- if let Some(node) = self.nodes[slot].as_mut() {- node.prev = None;- node.next = None;- }- }-- #[inline]- fn remove_slot_from_bucket(- &mut self,- size: usize,- _slot: usize,- bucket_idx: usize,- ) {- if let Some(slots) = self.size_index.get_mut(&size) {- let last_idx = slots.len() - 1;- if bucket_idx != last_idx {- let swapped_slot = slots[last_idx];- slots[bucket_idx] = swapped_slot;- if let Some(swapped_node) = self.nodes[swapped_slot].as_mut() {- swapped_node.bucket_idx = bucket_idx;- }- }- slots.pop();- if slots.is_empty() {- self.size_index.remove(&size);- }- }- }-- #[inline]- pub(super) fn find_buffer(- &mut self,- size: usize,- ) -> Option<(B, usize)> {- let max_size = (2 * size).min(size + 2 * PAGE_SIZE);-- let mut found_key = None;- let mut found_slot = None;- let mut found_bucket_idx = None;-- for (&key, slots) in self.size_index.range(size..=max_size) {- if let Some(&slot) = slots.last() {- found_key = Some(key);- found_slot = Some(slot);- found_bucket_idx = Some(slots.len() - 1);- break;- }- }-- let (key, slot, bucket_idx) = match (found_key, found_slot, found_bucket_idx) {- (Some(k), Some(s), Some(i)) => (k, s, i),- _ => return None,- };-- self.remove_slot_from_bucket(key, slot, bucket_idx);- self.remove_from_lru(slot);-- let node = self.nodes[slot].take()?;- self.free_slots.push(slot);- self.pool_size = self.pool_size.saturating_sub(key);-- Some((node.buffer, key))- }-- #[inline]- pub(super) fn return_buffer(- &mut self,- buffer: B,- size: usize,- ) {- let slot = self.allocate_slot();- let bucket_idx = self.size_index.get(&size).map_or(0, |v| v.len());-- self.nodes[slot] = Some(BufferNode {- buffer,- size,- bucket_idx,- prev: None,- next: None,- });-- self.add_to_lru_head(slot);- self.size_index.entry(size).or_default().push(slot);- self.pool_size += size;- }-- pub(super) fn release_cached_buffers(- &mut self,- min_bytes_to_free: usize,- ) -> usize {- if min_bytes_to_free == 0 {- return 0;- }-- if min_bytes_to_free >= (self.pool_size * 9) / 10 {- return self.clear();- }-- let mut total_freed = 0usize;- let mut buffers_released = 0usize;-- while total_freed < min_bytes_to_free {- let tail_slot = match self.lru_tail {- Some(s) => s,- None => break,- };-- let (size, bucket_idx) = match self.nodes[tail_slot].as_ref() {- Some(n) => (n.size, n.bucket_idx),- None => break,- };-- self.remove_slot_from_bucket(size, tail_slot, bucket_idx);- self.remove_from_lru(tail_slot);- self.nodes[tail_slot] = None;- self.free_slots.push(tail_slot);-- total_freed += size;- buffers_released += 1;- self.pool_size = self.pool_size.saturating_sub(size);- self.total_allocated = self.total_allocated.saturating_sub(size);- }-- buffers_released- }-- #[inline]- pub(super) fn available_size(&self) -> usize {- self.pool_size- }-- pub(super) fn clear(&mut self) -> usize {- let count = self.nodes.iter().filter(|n| n.is_some()).count();-- self.nodes.clear();- self.free_slots.clear();- self.size_index.clear();- self.lru_head = None;- self.lru_tail = None;- self.total_allocated = self.total_allocated.saturating_sub(self.pool_size);- self.pool_size = 0;-- count- }-}diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rsindex 9d811eb5c..e7bba72f3 100644--- a/crates/uzu/src/backends/common/backend.rs+++ b/crates/uzu/src/backends/common/backend.rs@@ -1,13 +1,11 @@use std::{error::Error, fmt::Debug};-use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer, kernel::matmul::MatmulKernels};+use super::{Buffer, CommandBuffer, Context, Event, Kernels, kernel::matmul::MatmulKernels};pub trait Backend: Debug + Clone + 'static {type Context: Context<Backend = Self>;- type NativeBuffer: NativeBuffer<Backend = Self>;type CommandBuffer: CommandBuffer<Backend = Self>;- type ComputeEncoder;- type CopyEncoder: CopyEncoder<Backend = Self>;+ type Buffer: Buffer<Backend = Self>;type Event: Event<Backend = Self>;type Kernels: Kernels<Backend = Self> + MatmulKernels;type Error: Error + Debug;diff --git a/crates/uzu/src/backends/common/buffer.rs b/crates/uzu/src/backends/common/buffer.rsindex 8630de880..832ce2cc9 100644--- a/crates/uzu/src/backends/common/buffer.rs+++ b/crates/uzu/src/backends/common/buffer.rs@@ -1,38 +1,18 @@-use std::{mem::ManuallyDrop, rc::Weak};+use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};-use super::{Backend, Context};+use super::Backend;-pub struct Buffer<B: Backend> {- inner: ManuallyDrop<B::NativeBuffer>,- context: Weak<B::Context>,- is_scratch: bool,-}+pub trait Buffer: Debug {+ type Backend: Backend<Buffer = Self>;-impl<B: Backend> Buffer<B> {- pub fn new(- inner: B::NativeBuffer,- context: Weak<B::Context>,- is_scratch: bool,- ) -> Self {- Self {- inner: ManuallyDrop::new(inner),- context,- is_scratch,- }- }+ fn set_label(+ &mut self,+ label: Option<&str>,+ );- pub fn inner(&self) -> &B::NativeBuffer {- &self.inner- }-}+ fn cpu_ptr(&self) -> NonNull<c_void>;++ fn length(&self) -> usize;-impl<B: Backend> Drop for Buffer<B> {- fn drop(&mut self) {- // Safety: drop is only called once, inner is valid- let inner = unsafe { ManuallyDrop::take(&mut self.inner) };- if let Some(ctx) = self.context.upgrade() {- ctx.allocator().handle_buffer_drop(inner, self.is_scratch);- }- // If context is gone, inner drops naturally here- }+ fn id(&self) -> usize;}diff --git a/crates/uzu/src/backends/common/buffer_lifetime.rs b/crates/uzu/src/backends/common/buffer_lifetime.rsdeleted file mode 100644index 65cf6b5a7..000000000--- a/crates/uzu/src/backends/common/buffer_lifetime.rs+++ /dev/null@@ -1,5 +0,0 @@-#[derive(Clone, Copy, Debug, PartialEq, Eq)]-pub enum BufferLifetime {- Permanent,- Scratch,-}diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex a23e00622..a9bdb96cc 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -1,38 +1,84 @@+use std::ops::Range;+use super::Backend;pub trait CommandBuffer {- type Backend: Backend;+ type Backend: Backend<CommandBuffer = Self>;++ type Initial: CommandBufferInitial<CommandBuffer = Self>;+ type Encoding: CommandBufferEncoding<CommandBuffer = Self>;+ type Executable: CommandBufferExecutable<CommandBuffer = Self>;+ type Pending: CommandBufferPending<CommandBuffer = Self>;+ type Completed: CommandBufferCompleted<CommandBuffer = Self>;+}++pub trait CommandBufferInitial {+ type CommandBuffer: CommandBuffer<Initial = Self>;- fn with_compute_encoder<T>(+ fn start_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Encoding;+}++pub trait CommandBufferEncoding {+ type CommandBuffer: CommandBuffer<Encoding = Self>;++ fn encode_copy(&mut self,- callback: impl FnOnce(&mut <Self::Backend as Backend>::ComputeEncoder) -> T,- ) -> T;+ src: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,+ dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,+ size: usize,+ );- fn with_copy_encoder<T>(+ fn encode_fill(&mut self,- callback: impl FnOnce(&mut <Self::Backend as Backend>::CopyEncoder) -> T,- ) -> T;+ dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,+ range: Range<usize>,+ value: u8,+ );fn encode_wait_for_event(&mut self,- event: &<Self::Backend as Backend>::Event,+ event: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Event,value: u64,);fn encode_signal_event(&mut self,- event: &<Self::Backend as Backend>::Event,+ event: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Event,value: u64,);fn add_completion_handler(&mut self,- handler: impl FnOnce(Result<(), <Self::Backend as Backend>::Error>) + 'static,+ handler: impl FnOnce(+ Result<+ &<Self::CommandBuffer as CommandBuffer>::Completed,+ <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,+ >,+ ) + 'static,);- fn submit(&mut self);+ fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;+}++pub trait CommandBufferExecutable {+ type CommandBuffer: CommandBuffer<Executable = Self>;++ fn submit(self) -> <Self::CommandBuffer as CommandBuffer>::Pending;+}++pub trait CommandBufferPending {+ type CommandBuffer: CommandBuffer<Pending = Self>;++ fn wait_until_completed(+ self+ ) -> Result<+ <Self::CommandBuffer as CommandBuffer>::Completed,+ <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,+ >;+}- fn wait_until_completed(&self) -> Result<(), <Self::Backend as Backend>::Error>;+pub trait CommandBufferCompleted {+ type CommandBuffer: CommandBuffer<Completed = Self>;fn gpu_execution_time_ms(&self) -> Option<f64>;}diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rsindex ca5a27cff..d4fb3fd05 100644--- a/crates/uzu/src/backends/common/context.rs+++ b/crates/uzu/src/backends/common/context.rs@@ -1,6 +1,7 @@use std::{path::Path, rc::Rc};-use super::{Allocator, Backend};+use super::Backend;+use crate::backends::common::CommandBuffer;#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum DeviceClass {@@ -26,16 +27,14 @@ pub trait Context: Sized {fn debug_active(&self) -> bool;- fn allocator(&self) -> &Allocator<Self::Backend>;+ fn create_command_buffer(+ &self+ ) -> Result<<<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Initial, <Self::Backend as Backend>::Error>;fn create_buffer(&self,size: usize,- ) -> Result<<Self::Backend as Backend>::NativeBuffer, <Self::Backend as Backend>::Error>;-- fn create_command_buffer(- &self- ) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;+ ) -> Result<<Self::Backend as Backend>::Buffer, <Self::Backend as Backend>::Error>;fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rsdeleted file mode 100644index 93c319ae2..000000000--- a/crates/uzu/src/backends/common/copy_encoder.rs+++ /dev/null@@ -1,21 +0,0 @@-use std::ops::Range;--use super::Backend;--pub trait CopyEncoder {- type Backend: Backend;-- fn encode_copy(- &mut self,- src: &<Self::Backend as Backend>::NativeBuffer,- dst: &mut <Self::Backend as Backend>::NativeBuffer,- size: usize,- );-- fn encode_fill(- &mut self,- dst: &mut <Self::Backend as Backend>::NativeBuffer,- range: Range<usize>,- value: u8,- );-}diff --git a/crates/uzu/src/backends/common/event.rs b/crates/uzu/src/backends/common/event.rsindex 57eeccf77..776b143cb 100644--- a/crates/uzu/src/backends/common/event.rs+++ b/crates/uzu/src/backends/common/event.rs@@ -1,5 +1,5 @@use super::Backend;pub trait Event {- type Backend: Backend;+ type Backend: Backend<Event = Self>;}diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rsindex 8d8650504..c0522bdd7 100644--- a/crates/uzu/src/backends/common/kernel/attention.rs+++ b/crates/uzu/src/backends/common/kernel/attention.rs@@ -6,7 +6,7 @@ use std::{use crate::{DataType,backends::common::{- Backend, Kernels,+ Backend, CommandBuffer, Kernels,gpu_types::{AttnMaskParams, AttnParams},kernel::AttentionGemmKernel,},@@ -15,12 +15,12 @@ use crate::{const BQ: usize = 32;pub struct AttentionGemmArguments<'a, B: Backend> {- pub queries_buffer: &'a B::NativeBuffer, // buffer(0)- pub keys_buffer: &'a B::NativeBuffer, // buffer(1)- pub values_buffer: &'a B::NativeBuffer, // buffer(2)- pub output_buffer: &'a mut B::NativeBuffer, // buffer(3)- pub mask_buffer: Option<&'a B::NativeBuffer>, // buffer(6)- pub sinks_buffer: Option<&'a B::NativeBuffer>, // buffer(7)+ pub queries_buffer: &'a B::Buffer, // buffer(0)+ pub keys_buffer: &'a B::Buffer, // buffer(1)+ pub values_buffer: &'a B::Buffer, // buffer(2)+ pub output_buffer: &'a mut B::Buffer, // buffer(3)+ pub mask_buffer: Option<&'a B::Buffer>, // buffer(6)+ pub sinks_buffer: Option<&'a B::Buffer>, // buffer(7)pub num_heads: usize,pub num_groups: usize,pub suffix_length: usize, // qL@@ -49,7 +49,7 @@ impl<B: Backend> AttentionGemmBlock<B> {pub fn encode(&self,context: &B::Context,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: AttentionGemmArguments<B>,) -> Result<(), B::Error> {let bk: usize = if args.head_dim < 128 {@@ -138,7 +138,7 @@ impl<B: Backend> AttentionGemmBlock<B> {args.sinks_buffer,args.num_heads as u32,args.suffix_length as u32,- compute_encoder,+ command_buffer,);Ok(())diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rsindex dbe4e942f..04692fa70 100644--- a/crates/uzu/src/backends/common/kernel/audio.rs+++ b/crates/uzu/src/backends/common/kernel/audio.rs@@ -1,4 +1,4 @@-use crate::backends::common::Backend;+use crate::backends::common::{Backend, CommandBuffer};#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum AudioPadMode {@@ -7,9 +7,9 @@ pub enum AudioPadMode {}pub struct AudioFsqDecodeArguments<'a, B: Backend> {- pub tokens: &'a B::NativeBuffer,- pub lengths: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,+ pub tokens: &'a B::Buffer,+ pub lengths: &'a B::Buffer,+ pub output: &'a B::Buffer,pub batch_size: usize,pub num_groups: usize,pub seq_len: usize,@@ -18,9 +18,9 @@ pub struct AudioFsqDecodeArguments<'a, B: Backend> {}pub struct AudioFsqEncodeArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub tokens: &'a B::NativeBuffer,- pub lengths: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub tokens: &'a B::Buffer,+ pub lengths: &'a B::Buffer,pub batch_size: usize,pub num_groups: usize,pub seq_len: usize,@@ -31,11 +31,11 @@ pub struct AudioFsqEncodeArguments<'a, B: Backend> {}pub struct AudioConv1dArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub weight: &'a B::NativeBuffer,- pub bias: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,- pub lengths: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub weight: &'a B::Buffer,+ pub bias: &'a B::Buffer,+ pub output: &'a B::Buffer,+ pub lengths: &'a B::Buffer,pub batch_size: usize,pub cin: usize,pub cout: usize,@@ -49,11 +49,11 @@ pub struct AudioConv1dArguments<'a, B: Backend> {}pub struct AudioCausalConv1dArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub weight: &'a B::NativeBuffer,- pub bias: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,- pub lengths: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub weight: &'a B::Buffer,+ pub bias: &'a B::Buffer,+ pub output: &'a B::Buffer,+ pub lengths: &'a B::Buffer,pub batch_size: usize,pub cin: usize,pub cout: usize,@@ -63,11 +63,11 @@ pub struct AudioCausalConv1dArguments<'a, B: Backend> {}pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub weight: &'a B::NativeBuffer,- pub bias: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,- pub lengths: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub weight: &'a B::Buffer,+ pub bias: &'a B::Buffer,+ pub output: &'a B::Buffer,+ pub lengths: &'a B::Buffer,pub batch_size: usize,pub cin: usize,pub cout: usize,@@ -78,9 +78,9 @@ pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {}pub struct AudioHalfSnakeArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub alpha: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub alpha: &'a B::Buffer,+ pub output: &'a B::Buffer,pub batch_size: usize,pub channels: usize,pub seq_len: usize,@@ -90,28 +90,28 @@ pub struct AudioHalfSnakeArguments<'a, B: Backend> {}pub struct AudioElementwiseArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub output: &'a B::Buffer,pub n: usize,}pub struct AudioAddArguments<'a, B: Backend> {- pub a: &'a B::NativeBuffer,- pub b: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,+ pub a: &'a B::Buffer,+ pub b: &'a B::Buffer,+ pub output: &'a B::Buffer,pub n: usize,}pub struct AudioScaleArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub output: &'a B::Buffer,pub n: usize,pub scale: f32,}pub struct AudioClampArguments<'a, B: Backend> {- pub input: &'a B::NativeBuffer,- pub output: &'a B::NativeBuffer,+ pub input: &'a B::Buffer,+ pub output: &'a B::Buffer,pub n: usize,pub min_value: f32,pub max_value: f32,@@ -120,68 +120,68 @@ pub struct AudioClampArguments<'a, B: Backend> {pub trait AudioKernelRuntime<B: Backend> {fn encode_fsq_decode(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioFsqDecodeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_fsq_encode(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioFsqEncodeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_conv1d(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioConv1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_causal_conv1d(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioCausalConv1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_causal_conv_transpose1d(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioCausalConvTranspose1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_half_snake(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioHalfSnakeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_leaky_relu(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioElementwiseArguments<'_, B>,negative_slope: f32,) -> Result<(), B::Error>;fn encode_tanh(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioElementwiseArguments<'_, B>,) -> Result<(), B::Error>;fn encode_add(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioAddArguments<'_, B>,) -> Result<(), B::Error>;fn encode_scale(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioScaleArguments<'_, B>,) -> Result<(), B::Error>;fn encode_clamp(&self,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: AudioClampArguments<'_, B>,) -> Result<(), B::Error>;}diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsindex bf6a43bae..def3ce934 100644--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -8,9 +8,9 @@ use crate::{};pub struct KVLayerData<B: Backend> {- pub key_buffer: Rc<RefCell<B::NativeBuffer>>,+ pub key_buffer: Rc<RefCell<B::Buffer>>,pub key_shape: [usize; 3],- pub value_buffer: Rc<RefCell<B::NativeBuffer>>,+ pub value_buffer: Rc<RefCell<B::Buffer>>,pub value_shape: [usize; 3],}@@ -52,25 +52,13 @@ impl<B: Backend> KVCacheUpdate<B> {})}+ /// Encode the KV cache update operation using a provided command bufferpub fn encode(&self,in_place_data: &[KVLayerData<B>],source_indices: &[usize],destination_indices: &[usize],- command_buffer: &mut B::CommandBuffer,- ) -> Result<(), KVCacheUpdateError<B>> {- command_buffer.with_compute_encoder(|encoder| {- self.encode_with_encoder(in_place_data, source_indices, destination_indices, encoder)- })- }-- /// Encode the KV cache update operation using a provided compute encoder- pub fn encode_with_encoder(- &self,- in_place_data: &[KVLayerData<B>],- source_indices: &[usize],- destination_indices: &[usize],- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), KVCacheUpdateError<B>> {if source_indices.len() != destination_indices.len() {return Err(KVCacheUpdateError::IndicesCountMismatch);@@ -99,7 +87,7 @@ impl<B: Backend> KVCacheUpdate<B> {num_heads as u32,max_sequence_length as u32,head_dim as u32,- encoder,+ command_buffer,);}}diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rsindex ad3abda7f..ec91a6d71 100644--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs@@ -1,12 +1,15 @@use super::{MatmulError, MatmulKernels};-use crate::{DataType, backends::common::Backend};+use crate::{+ DataType,+ backends::common::{Backend, CommandBuffer},+};pub struct FullPrecisionMatmulArguments<'a, B: Backend> {- pub a: &'a B::NativeBuffer,+ pub a: &'a B::Buffer,pub a_offset: usize,- pub b: &'a B::NativeBuffer,- pub output: &'a mut B::NativeBuffer,- pub bias: Option<&'a B::NativeBuffer>,+ pub b: &'a B::Buffer,+ pub output: &'a mut B::Buffer,+ pub bias: Option<&'a B::Buffer>,pub batch: usize,pub input_dim: usize,pub output_dim: usize,@@ -23,7 +26,7 @@ pub trait FullPrecisionMatmulKernel: Sized {fn encode(&mut self,context: &<Self::Backend as Backend>::Context,- encoder: &mut <Self::Backend as Backend>::ComputeEncoder,+ command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,arguments: FullPrecisionMatmulArguments<Self::Backend>,);}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rsindex 4c78c8d84..ca3df3b48 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs@@ -6,7 +6,7 @@ use super::{use crate::{DataType,backends::common::{- Backend, Kernels,+ Backend, CommandBuffer, Kernels,kernel::{MatmulGemmKernel, matmul::MatmulError},},};@@ -66,7 +66,7 @@ impl<B: Backend> GemmKernel<B> {context: &B::Context,arguments: &mut MatmulArguments<B>,dispatch_descriptor: &DispatchDescriptor,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {let config = dispatch_descriptor.specialization;@@ -86,7 +86,7 @@ impl<B: Backend> GemmKernel<B> {group_count_x,group_count_y,group_count_z,- encoder,+ command_buffer,);Ok(())diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rsindex 2a3a83c15..e4046f267 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs@@ -8,7 +8,7 @@ use super::{use crate::{DataType,backends::common::{- Backend, Kernels,+ Backend, CommandBuffer, Kernels,kernel::{MatmulGemvKernel, matmul::MatmulError},},};@@ -67,7 +67,7 @@ impl<B: Backend> GemvKernel<B> {context: &B::Context,arguments: &mut MatmulArguments<B>,dispatch_descriptor: &DispatchDescriptor,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {let config = dispatch_descriptor.specialization;let pipeline = self.get_or_create_kernel(context, config)?;@@ -118,7 +118,7 @@ impl<B: Backend> GemvKernel<B> {dispatch_descriptor.bias_stride,dispatch_descriptor.batch_rows,config.output_rows_per_threadgroup() as i32,- encoder,+ command_buffer,);Ok(())diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rsindex c1c8d0954..e021bf13c 100644--- a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs@@ -6,7 +6,7 @@ use super::{};use crate::{DataType,- backends::common::{Backend, Kernels, kernel::TensorAddBiasKernel},+ backends::common::{Backend, CommandBuffer, Kernels, kernel::TensorAddBiasKernel},};pub struct MatmulKernel<B: Backend> {@@ -74,20 +74,20 @@ impl<B: Backend> MatmulKernel<B> {context: &B::Context,arguments: &mut MatmulArguments<B>,dispatch_descriptor: &MatmulDispatchDescriptor,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {match dispatch_descriptor {MatmulDispatchDescriptor::Gemv(d) => {let gemv = self.get_or_create_gemv()?;- gemv.encode(context, arguments, d, encoder)+ gemv.encode(context, arguments, d, command_buffer)},MatmulDispatchDescriptor::SplitK(d) => {let splitk = self.get_or_create_splitk()?;- splitk.encode(context, arguments, d, encoder)+ splitk.encode(context, arguments, d, command_buffer)},MatmulDispatchDescriptor::Gemm(d) => {let gemm = self.get_or_create_gemm()?;- gemm.encode(context, arguments, d, encoder)+ gemm.encode(context, arguments, d, command_buffer)},}}@@ -97,13 +97,13 @@ impl<B: Backend> MatmulKernel<B> {context: &B::Context,mut arguments: MatmulArguments<B>,dispatch_descriptor: &MatmulDispatchDescriptor,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {- self.encode_dispatch_descriptor(context, &mut arguments, dispatch_descriptor, encoder)?;+ self.encode_dispatch_descriptor(context, &mut arguments, dispatch_descriptor, command_buffer)?;if let Some(bias) = arguments.bias {if !dispatch_descriptor.bias_is_fused() {- self.apply_bias_add(context, &mut arguments, bias, encoder)?;+ self.apply_bias_add(context, &mut arguments, bias, command_buffer)?;}}@@ -114,8 +114,8 @@ impl<B: Backend> MatmulKernel<B> {&mut self,context: &B::Context,arguments: &mut MatmulArguments<B>,- bias: &B::NativeBuffer,- encoder: &mut B::ComputeEncoder,+ bias: &B::Buffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {let m = arguments.batch as usize;let n = arguments.output_dim as usize;@@ -132,7 +132,7 @@ impl<B: Backend> MatmulKernel<B> {);}let bias_add = self.bias_add.as_ref().unwrap();- bias_add.encode(None::<&B::NativeBuffer>, bias, arguments.d.deref_mut(), n as u32, total_len as u32, encoder);+ bias_add.encode(None::<&B::Buffer>, bias, arguments.d.deref_mut(), n as u32, total_len as u32, command_buffer);Ok(())}diff --git a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rsindex ce4e478c1..49c60a860 100644--- a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs@@ -2,12 +2,12 @@ use crate::backends::common::Backend;#[derive(Debug)]pub struct MatmulArguments<'a, B: Backend> {- pub a: &'a B::NativeBuffer,+ pub a: &'a B::Buffer,/// Byte offset into `a` (used for slicing the batch dimension).pub a_offset: u64,- pub b: &'a B::NativeBuffer,- pub d: &'a mut B::NativeBuffer,- pub bias: Option<&'a B::NativeBuffer>,+ pub b: &'a B::Buffer,+ pub d: &'a mut B::Buffer,+ pub bias: Option<&'a B::Buffer>,/// M dimension - batch/number of tokens (rows of A, rows of D)pub batch: i32,/// K dimension - input_dim/reduction dimension (cols of A, rows of B)diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rsindex 0116066c1..08cd9b9d2 100644--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs@@ -4,7 +4,7 @@ use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::Dispause crate::{DataType,backends::common::{- Backend, Context, Kernels,+ Backend, CommandBuffer, Context, Kernels,kernel::{MatmulSplitKAccumBfloat16Kernel, MatmulSplitKPartialBfloat16Kernel, matmul::MatmulError},},};@@ -13,7 +13,7 @@ pub struct SplitKKernel<B: Backend> {data_type: DataType,partial_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKPartialBfloat16Kernel>,accum_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKAccumBfloat16Kernel>,- accumulator_buffer: Option<B::NativeBuffer>,+ accumulator_buffer: Option<B::Buffer>,accumulator_buffer_bytes: usize,}@@ -68,7 +68,7 @@ impl<B: Backend> SplitKKernel<B> {context: &B::Context,arguments: &mut MatmulArguments<B>,dispatch_descriptor: &DispatchDescriptor,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), MatmulError<B>> {self.ensure_kernels(context)?;self.ensure_accumulator_buffer(context, dispatch_descriptor.accumulator_bytes)?;@@ -89,7 +89,7 @@ impl<B: Backend> SplitKKernel<B> {partial_group_count_x,partial_group_count_y,partial_group_count_z,- encoder,+ command_buffer,);let accum_total_threads_x = u32::try_from(dispatch_descriptor.accum_total_threads.x)@@ -105,7 +105,7 @@ impl<B: Backend> SplitKKernel<B> {arguments.ldd,accum_total_threads_x,accum_total_threads_y,- encoder,+ command_buffer,);Ok(())diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rsindex 3e56c2318..6e83fefa2 100644--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -1,6 +1,6 @@use crate::{DataType,- backends::common::{Backend, Kernels, kernel::MlpGateActMulKernel},+ backends::common::{Backend, CommandBuffer, Kernels, kernel::MlpGateActMulKernel},config::Activation,};@@ -27,9 +27,9 @@ impl<B: Backend> MlpGateActMulEncodable<B> {pub fn encode(&self,- encoder: &mut B::ComputeEncoder,- fused_up: &B::NativeBuffer,- hidden: &mut B::NativeBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ fused_up: &B::Buffer,+ hidden: &mut B::Buffer,m: i32,) -> Result<(), B::Error> {let act_type = match self.activation {@@ -41,7 +41,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {panic!("Identity activation is not supported for kernel")},};- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, command_buffer);Ok(())}}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rsindex f2d3d84f0..bcaa38a61 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs@@ -14,23 +14,23 @@ static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];#[derive(Debug)]pub struct MoeExpertsSingleDecodeArguments<'a, B: Backend> {/// Input activation [d_model]- pub x: &'a B::NativeBuffer,+ pub x: &'a B::Buffer,/// Top-K expert indices from router [K]- pub topk_ids: &'a B::NativeBuffer,+ pub topk_ids: &'a B::Buffer,/// Top-K probabilities from router [K]- pub topk_probs: &'a B::NativeBuffer,+ pub topk_probs: &'a B::Buffer,/// Up/gate projection weights [E, 2*d_ff, d_model]- pub w13_all: &'a B::NativeBuffer,+ pub w13_all: &'a B::Buffer,/// Down projection weights [E, d_model, d_ff]- pub w2_all: &'a B::NativeBuffer,+ pub w2_all: &'a B::Buffer,/// Up/gate biases [E, 2*d_ff]- pub up_biases: &'a B::NativeBuffer,+ pub up_biases: &'a B::Buffer,/// Down biases [E, d_model]- pub down_biases: &'a B::NativeBuffer,+ pub down_biases: &'a B::Buffer,/// Hidden buffer [K, d_ff] - intermediate storage (f32)- pub hidden: &'a mut B::NativeBuffer,+ pub hidden: &'a mut B::Buffer,/// Final output [d_model]- pub y: &'a mut B::NativeBuffer,+ pub y: &'a mut B::Buffer,/// Model dimensionpub d_model: usize,/// FFN hidden dimension@@ -84,7 +84,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {pub fn encode(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,mut args: MoeExpertsSingleDecodeArguments<B>,) {if args.k == 0 {@@ -95,41 +95,37 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();// Pass A: x @ W13[expert] -> hidden- command_buffer.with_compute_encoder(|encoder| {- let kernel = &self.pass_a[gate_idx][dtype_idx];- kernel.encode(- args.x,- args.topk_ids,- args.w13_all,- args.up_biases,- args.hidden.deref_mut(),- args.d_model as u32,- args.d_ff as u32,- args.k as u32,- args.silu_alpha,- args.gate_clip_min,- args.gate_clip_max,- args.up_clip_min,- args.up_clip_max,- encoder,- );- });+ let kernel = &self.pass_a[gate_idx][dtype_idx];+ kernel.encode(+ args.x,+ args.topk_ids,+ args.w13_all,+ args.up_biases,+ args.hidden.deref_mut(),+ args.d_model as u32,+ args.d_ff as u32,+ args.k as u32,+ args.silu_alpha,+ args.gate_clip_min,+ args.gate_clip_max,+ args.up_clip_min,+ args.up_clip_max,+ command_buffer,+ );// Pass B: 8 simdgroups (256 threads), outputs final y directly- command_buffer.with_compute_encoder(|encoder| {- let kernel = &self.pass_b[dtype_idx];- kernel.encode(- args.hidden.deref(),- args.topk_ids,- args.topk_probs,- args.w2_all,- args.down_biases,- args.y,- args.d_model as u32,- args.d_ff as u32,- args.k as u32,- encoder,- );- });+ let kernel = &self.pass_b[dtype_idx];+ kernel.encode(+ args.hidden.deref(),+ args.topk_ids,+ args.topk_probs,+ args.w2_all,+ args.down_biases,+ args.y,+ args.d_model as u32,+ args.d_ff as u32,+ args.k as u32,+ command_buffer,+ );}}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rsindex 7b7adfcde..1030d6a5c 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs@@ -7,8 +7,8 @@ use super::{use crate::{DataType,backends::common::{- CommandBuffer,- kernel::{Backend, Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},+ Backend, CommandBuffer,+ kernel::{Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},},};@@ -47,7 +47,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {pub fn encode(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,mut args: MoeExpertsTwoPassArguments<B>,) {if args.total_rows == 0 {@@ -108,43 +108,39 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();// pass a- command_buffer.with_compute_encoder(|encoder| {- let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];- pass_a_kernel.encode(- args.x_perm_buffer,- args.expert_offsets,- args.w13_all,- args.hidden_buffer.deref_mut(),- args.up_biases,- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- args.gate_clip_min,- args.gate_clip_max,- args.up_clip_min,- args.up_clip_max,- args.silu_alpha,- args.tile_map.deref(),- args.dispatch_args.deref(),- encoder,- );- });+ let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];+ pass_a_kernel.encode(+ args.x_perm_buffer,+ args.expert_offsets,+ args.w13_all,+ args.hidden_buffer.deref_mut(),+ args.up_biases,+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ args.gate_clip_min,+ args.gate_clip_max,+ args.up_clip_min,+ args.up_clip_max,+ args.silu_alpha,+ args.tile_map.deref(),+ args.dispatch_args.deref(),+ command_buffer,+ );// pass b- command_buffer.with_compute_encoder(|encoder| {- let pass_b_kernel = &self.fused_down[dtype_idx];- pass_b_kernel.encode(- args.hidden_buffer.deref(),- args.row_expert_map.deref(),- args.w2_all,- args.down_biases,- args.output_buffer.deref_mut(),- args.total_rows as u32,- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- encoder,- );- });+ let pass_b_kernel = &self.fused_down[dtype_idx];+ pass_b_kernel.encode(+ args.hidden_buffer.deref(),+ args.row_expert_map.deref(),+ args.w2_all,+ args.down_biases,+ args.output_buffer.deref_mut(),+ args.total_rows as u32,+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ command_buffer,+ );}}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rsindex 10c3f74d6..15cd2e280 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs@@ -6,7 +6,7 @@ use super::{use crate::{DataType,backends::common::{- Backend, CommandBuffer, CopyEncoder, Kernels,+ Backend, CommandBuffer, CommandBufferEncoding, Kernels,kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},},};@@ -48,7 +48,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {pub fn encode(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,mut args: MoeExpertsTwoPassArguments<B>,) {if args.total_rows == 0 {@@ -56,8 +56,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {}let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();- command_buffer- .with_copy_encoder(|encoder| encoder.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0));+ command_buffer.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);self.tile_map.encode_counts(command_buffer,@@ -112,27 +111,25 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {let gate_idx = args.gating_code.min(3) as usize;let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();- command_buffer.with_compute_encoder(|encoder| {- let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];- kernel_pass_a.encode(- args.x_perm_buffer,- args.expert_offsets,- args.w13_all,- args.up_biases,- args.hidden_buffer.deref_mut(),- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- args.gate_clip_min,- args.gate_clip_max,- args.up_clip_min,- args.up_clip_max,- args.silu_alpha,- args.tile_map.deref(),- args.dispatch_args.deref(),- encoder,- );- });+ let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];+ kernel_pass_a.encode(+ args.x_perm_buffer,+ args.expert_offsets,+ args.w13_all,+ args.up_biases,+ args.hidden_buffer.deref_mut(),+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ args.gate_clip_min,+ args.gate_clip_max,+ args.up_clip_min,+ args.up_clip_max,+ args.silu_alpha,+ args.tile_map.deref(),+ args.dispatch_args.deref(),+ command_buffer,+ );let dispatch_args = MoeTileDispatchArguments {total_tiles: args.total_tiles,@@ -141,41 +138,39 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {};self.tile_map.encode_dispatch_args(command_buffer, dispatch_args);- command_buffer.with_compute_encoder(|encoder| {- let kernel_pass_b = &self.pass_b_indirect[dtype_idx];- kernel_pass_b.encode(- args.hidden_buffer.deref(),- args.expert_offsets,- args.w2_all,- args.down_biases,- args.output_buffer.deref_mut(),- args.d_model as u32,- args.d_ff as u32,- args.e as u32,- args.tile_map.deref(),- args.dispatch_args.deref(),- encoder,- );- });+ let kernel_pass_b = &self.pass_b_indirect[dtype_idx];+ kernel_pass_b.encode(+ args.hidden_buffer.deref(),+ args.expert_offsets,+ args.w2_all,+ args.down_biases,+ args.output_buffer.deref_mut(),+ args.d_model as u32,+ args.d_ff as u32,+ args.e as u32,+ args.tile_map.deref(),+ args.dispatch_args.deref(),+ command_buffer,+ );}}#[derive(Debug)]pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {- pub x_perm_buffer: &'a B::NativeBuffer,- pub expert_offsets: &'a B::NativeBuffer,- pub row_expert_map: &'a mut B::NativeBuffer,- pub hidden_buffer: &'a mut B::NativeBuffer,- pub output_buffer: &'a mut B::NativeBuffer,- pub w13_all: &'a B::NativeBuffer,- pub w2_all: &'a B::NativeBuffer,- pub up_biases: &'a B::NativeBuffer,- pub down_biases: &'a B::NativeBuffer,- pub tile_counts: &'a mut B::NativeBuffer,- pub tile_offsets: &'a mut B::NativeBuffer,- pub tile_map: &'a mut B::NativeBuffer,- pub total_tiles: &'a mut B::NativeBuffer,- pub dispatch_args: &'a mut B::NativeBuffer,+ pub x_perm_buffer: &'a B::Buffer,+ pub expert_offsets: &'a B::Buffer,+ pub row_expert_map: &'a mut B::Buffer,+ pub hidden_buffer: &'a mut B::Buffer,+ pub output_buffer: &'a mut B::Buffer,+ pub w13_all: &'a B::Buffer,+ pub w2_all: &'a B::Buffer,+ pub up_biases: &'a B::Buffer,+ pub down_biases: &'a B::Buffer,+ pub tile_counts: &'a mut B::Buffer,+ pub tile_offsets: &'a mut B::Buffer,+ pub tile_map: &'a mut B::Buffer,+ pub total_tiles: &'a mut B::Buffer,+ pub dispatch_args: &'a mut B::Buffer,pub total_rows: usize,pub d_model: usize,pub d_ff: usize,diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rsindex 38a4dd545..771c32ada 100644--- a/crates/uzu/src/backends/common/kernel/moe/gather.rs+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs@@ -8,10 +8,10 @@ use crate::{#[derive(Debug)]pub struct MoeGatherArguments<'a, B: Backend> {- pub x_buffer: &'a B::NativeBuffer,- pub bucketed_ids_buffer: &'a B::NativeBuffer,- pub x_perm_buffer: &'a mut B::NativeBuffer,- pub sumk_buffer: &'a B::NativeBuffer,+ pub x_buffer: &'a B::Buffer,+ pub bucketed_ids_buffer: &'a B::Buffer,+ pub x_perm_buffer: &'a mut B::Buffer,+ pub sumk_buffer: &'a B::Buffer,pub t: usize,pub k: usize,pub d_model: usize,@@ -34,11 +34,11 @@ impl<B: Backend> MoeGatherKernels<B> {pub fn encode(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,dtype: DataType,args: MoeGatherArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| match dtype {+ match dtype {DataType::F32 => self.f32.encode(args.x_buffer,args.bucketed_ids_buffer,@@ -47,7 +47,7 @@ impl<B: Backend> MoeGatherKernels<B> {args.d_model as u32,args.t as u32,args.k as u32,- encoder,+ command_buffer,),DataType::F16 => self.f16.encode(args.x_buffer,@@ -57,7 +57,7 @@ impl<B: Backend> MoeGatherKernels<B> {args.d_model as u32,args.t as u32,args.k as u32,- encoder,+ command_buffer,),DataType::BF16 => self.bf16.encode(args.x_buffer,@@ -67,9 +67,9 @@ impl<B: Backend> MoeGatherKernels<B> {args.d_model as u32,args.t as u32,args.k as u32,- encoder,+ command_buffer,),_ => panic!("Unsupported data type: {:?}", dtype),- });+ };}}diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rsindex fb5712cd2..ea144b317 100644--- a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs@@ -22,83 +22,75 @@ impl<B: Backend> MoeTileMapKernels<B> {pub fn encode_counts(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoeTileCountsArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);- });+ self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, command_buffer);}pub fn encode_scan(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoeTileScanArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.scan.encode(- args.tile_counts_buffer,- args.tile_offsets_buffer,- args.total_tiles_buffer,- args.e as u32,- encoder,- );- });+ self.scan.encode(+ args.tile_counts_buffer,+ args.tile_offsets_buffer,+ args.total_tiles_buffer,+ args.e as u32,+ command_buffer,+ );}pub fn encode_build_map(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoeTileMapBuildArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.build.encode(- args.expert_offsets,- args.tile_offsets,- args.tile_counts,- args.tile_map,- args.e as u32,- encoder,- );- });+ self.build.encode(+ args.expert_offsets,+ args.tile_offsets,+ args.tile_counts,+ args.tile_map,+ args.e as u32,+ command_buffer,+ );}pub fn encode_dispatch_args(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoeTileDispatchArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);- });+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, command_buffer);}}pub struct MoeTileCountsArguments<'a, B: Backend> {- pub offsets_buffer: &'a B::NativeBuffer, // [E+1]- pub tile_counts_buffer: &'a mut B::NativeBuffer, // [E]+ pub offsets_buffer: &'a B::Buffer, // [E+1]+ pub tile_counts_buffer: &'a mut B::Buffer, // [E]pub e: usize,}pub struct MoeTileScanArguments<'a, B: Backend> {- pub tile_counts_buffer: &'a B::NativeBuffer, // [E]- pub tile_offsets_buffer: &'a mut B::NativeBuffer, // [E+1]- pub total_tiles_buffer: &'a mut B::NativeBuffer, // [>=2]+ pub tile_counts_buffer: &'a B::Buffer, // [E]+ pub tile_offsets_buffer: &'a mut B::Buffer, // [E+1]+ pub total_tiles_buffer: &'a mut B::Buffer, // [>=2]pub e: usize,}#[derive(Debug)]pub struct MoeTileMapBuildArguments<'a, B: Backend> {- pub expert_offsets: &'a B::NativeBuffer, // [E+1]- pub tile_offsets: &'a B::NativeBuffer, // [E+1]- pub tile_counts: &'a B::NativeBuffer, // [E]- pub tile_map: &'a mut B::NativeBuffer, // [total_tiles * 3]+ pub expert_offsets: &'a B::Buffer, // [E+1]+ pub tile_offsets: &'a B::Buffer, // [E+1]+ pub tile_counts: &'a B::Buffer, // [E]+ pub tile_map: &'a mut B::Buffer, // [total_tiles * 3]pub e: usize,}#[derive(Debug)]pub struct MoeTileDispatchArguments<'a, B: Backend> {- pub total_tiles: &'a B::NativeBuffer, // [>=1]- pub dispatch_args: &'a mut B::NativeBuffer, // [3]- pub num_tiles_x: u32, // x dimension for indirect dispatch+ pub total_tiles: &'a B::Buffer, // [>=1]+ pub dispatch_args: &'a mut B::Buffer, // [3]+ pub num_tiles_x: u32, // x dimension for indirect dispatch}diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rsindex 8704a7e03..22353c53c 100644--- a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs@@ -1,49 +1,49 @@use crate::backends::common::{- CommandBuffer,+ Backend, CommandBuffer,kernel::{- Backend, Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,+ Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,},};#[derive(Debug)]pub struct MoePassATileCountsArguments<'a, B: Backend> {- pub expert_offsets: &'a B::NativeBuffer, // [E+1]- pub tile_counts: &'a mut B::NativeBuffer, // [E]+ pub expert_offsets: &'a B::Buffer, // [E+1]+ pub tile_counts: &'a mut B::Buffer, // [E]pub e: usize,pub h_blocks: u32,}#[derive(Debug)]pub struct MoePassATileScanArguments<'a, B: Backend> {- pub tile_counts: &'a B::NativeBuffer, // [E]- pub tile_offsets: &'a mut B::NativeBuffer, // [E+1]- pub total_tiles: &'a mut B::NativeBuffer, // [>=1]+ pub tile_counts: &'a B::Buffer, // [E]+ pub tile_offsets: &'a mut B::Buffer, // [E+1]+ pub total_tiles: &'a mut B::Buffer, // [>=1]pub e: usize,}#[derive(Debug)]pub struct MoePassARowMapArguments<'a, B: Backend> {- pub expert_offsets: &'a B::NativeBuffer, // [E+1]- pub row_expert_map: &'a mut B::NativeBuffer, // [total_rows]+ pub expert_offsets: &'a B::Buffer, // [E+1]+ pub row_expert_map: &'a mut B::Buffer, // [total_rows]pub total_rows: usize,pub e: usize,}#[derive(Debug)]pub struct MoePassATileBuildArguments<'a, B: Backend> {- pub expert_offsets: &'a B::NativeBuffer, // [E+1]- pub tile_offsets: &'a B::NativeBuffer, // [E+1]- pub row_expert_map: &'a B::NativeBuffer, // [total_rows]- pub tile_map: &'a mut B::NativeBuffer, // [total_tiles * 3]+ pub expert_offsets: &'a B::Buffer, // [E+1]+ pub tile_offsets: &'a B::Buffer, // [E+1]+ pub row_expert_map: &'a B::Buffer, // [total_rows]+ pub tile_map: &'a mut B::Buffer, // [total_tiles * 3]pub total_rows: usize,pub h_blocks: u32,}#[derive(Debug)]pub struct MoePassATileDispatchArguments<'a, B: Backend> {- pub total_tiles: &'a B::NativeBuffer, // [>=1]- pub dispatch_args: &'a mut B::NativeBuffer, // [3]+ pub total_tiles: &'a B::Buffer, // [>=1]+ pub dispatch_args: &'a mut B::Buffer, // [3]pub num_tiles_y: u32,}@@ -68,65 +68,55 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_counts(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoePassATileCountsArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);- });+ self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, command_buffer);}pub fn encode_scan(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoePassATileScanArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);- });+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, command_buffer);}pub fn encode_row_map(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoePassARowMapArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.row_map.encode(- args.expert_offsets,- args.row_expert_map,- args.total_rows as u32,- args.e as u32,- encoder,- );- });+ self.row_map.encode(+ args.expert_offsets,+ args.row_expert_map,+ args.total_rows as u32,+ args.e as u32,+ command_buffer,+ );}pub fn encode_build_map(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoePassATileBuildArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.build_map.encode(- args.expert_offsets,- args.tile_offsets,- args.row_expert_map,- args.tile_map,- args.total_rows as u32,- args.h_blocks,- encoder,- );- });+ self.build_map.encode(+ args.expert_offsets,+ args.tile_offsets,+ args.row_expert_map,+ args.tile_map,+ args.total_rows as u32,+ args.h_blocks,+ command_buffer,+ );}pub fn encode_dispatch_args(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: MoePassATileDispatchArguments<B>,) {- command_buffer.with_compute_encoder(|encoder| {- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);- });+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, command_buffer);}}diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex 6358aa83f..40daa27df 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -3,7 +3,7 @@ use std::collections::HashMap;use crate::{DataType,backends::common::{- Backend, Kernels,+ Backend, CommandBuffer, Kernels,kernel::{QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,@@ -54,12 +54,12 @@ pub struct QuantizedMatmulConfiguration {}pub struct QuantizedMatmulArguments<'a, B: Backend> {- pub a_buffer: &'a B::NativeBuffer,+ pub a_buffer: &'a B::Buffer,pub a_offset: usize,- pub b_buffer: &'a B::NativeBuffer,- pub scales_buffer: &'a B::NativeBuffer,- pub zero_points_or_biases_buffer: &'a B::NativeBuffer,- pub output_buffer: &'a mut B::NativeBuffer,+ pub b_buffer: &'a B::Buffer,+ pub scales_buffer: &'a B::Buffer,+ pub zero_points_or_biases_buffer: &'a B::Buffer,+ pub output_buffer: &'a mut B::Buffer,pub batch: usize,pub input_dim: usize,pub output_dim: usize,@@ -161,7 +161,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {pub fn encode(&self,- encoder: &mut B::ComputeEncoder,+ encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,arguments: QuantizedMatmulArguments<B>,) -> Result<(), QuantizedMatmulError<B>> {if arguments.quantization_type != self.quantization_type {@@ -487,9 +487,9 @@ fn to_i32<B: Backend>(}fn quant_buffers<'a, B: Backend>(- buffer: &'a B::NativeBuffer,+ buffer: &'a B::Buffer,quantization_type: QuantizedMatmulType,-) -> (Option<&'a B::NativeBuffer>, Option<&'a B::NativeBuffer>) {+) -> (Option<&'a B::Buffer>, Option<&'a B::Buffer>) {match quantization_type {QuantizedMatmulType::ZeroPoint => (Some(buffer), None),QuantizedMatmulType::Mlx => (None, Some(buffer)),diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rsindex 4a651fd50..921b4b5a9 100644--- a/crates/uzu/src/backends/common/kernel/sampling.rs+++ b/crates/uzu/src/backends/common/kernel/sampling.rs@@ -6,11 +6,10 @@ use std::{use thiserror::Error;-use super::{Backend, Kernels};use crate::{DataType,backends::common::{- Context,+ Backend, CommandBuffer, Context, Kernels,gpu_types::ArgmaxPair,kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,@@ -33,7 +32,7 @@ enum ArgmaxImplementation<B: Backend> {TwoPass {main_kernel: <B::Kernels as Kernels>::ArgmaxMainKernel,final_kernel: <B::Kernels as Kernels>::ArgmaxFinalKernel,- partial_results_buffer: RefCell<B::NativeBuffer>,+ partial_results_buffer: RefCell<B::Buffer>,},}@@ -142,18 +141,18 @@ impl<B: Backend> SamplingKernel<B> {})}- pub fn encode_with_encoder(+ pub fn encode(&self,- mut logits_buffer: &mut B::NativeBuffer,- seeds_buffer: Option<&B::NativeBuffer>,+ mut logits_buffer: &mut B::Buffer,+ seeds_buffer: Option<&B::Buffer>,seeds_offset: usize,- bitmask_buffer: Option<&B::NativeBuffer>,+ bitmask_buffer: Option<&B::Buffer>,bitmask_offset: usize,- sampled_tokens_buffer: &mut B::NativeBuffer,+ sampled_tokens_buffer: &mut B::Buffer,sampling_method: SamplingMethod,batch_size: usize,vocab_size: usize,- compute_encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), SamplingError<B>> {if batch_size > self.max_batch_size {return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));@@ -164,12 +163,12 @@ impl<B: Backend> SamplingKernel<B> {if let Some(bitmask_buffer) = bitmask_buffer {self.bitmask.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,(bitmask_buffer, bitmask_offset),logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,- compute_encoder,+ command_buffer,);}@@ -182,55 +181,55 @@ impl<B: Backend> SamplingKernel<B> {{if let Some(temperature) = temperature {self.temperature.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,temperature,- compute_encoder,+ command_buffer,);}if let Some(top_k) = top_k {self.topk.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,top_k,- compute_encoder,+ command_buffer,);}if let Some(top_p) = top_p {self.topp.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,top_p,- compute_encoder,+ command_buffer,);}if let Some(min_p) = min_p {self.minp.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,min_p,- compute_encoder,+ command_buffer,);}self.gumbel.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,(seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,- compute_encoder,+ command_buffer,);}@@ -243,7 +242,7 @@ impl<B: Backend> SamplingKernel<B> {sampled_tokens_buffer,batch_size as u32,vocab_size as u32,- compute_encoder,+ command_buffer,);},ArgmaxImplementation::TwoPass {@@ -256,14 +255,14 @@ impl<B: Backend> SamplingKernel<B> {partial_results_buffer.borrow_mut().deref_mut(),batch_size as u32,vocab_size as u32,- compute_encoder,+ command_buffer,);final_kernel.encode(partial_results_buffer.borrow().deref(),sampled_tokens_buffer,batch_size as u32,vocab_size as u32,- compute_encoder,+ command_buffer,);},}diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rsindex 20c5da315..cbd8a3f28 100644--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs@@ -1,7 +1,7 @@use crate::{DataType,backends::common::{- Backend, Kernels,+ Backend, CommandBuffer, Kernels,kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},},};@@ -13,14 +13,14 @@ pub enum SSDPrefillMode {}pub struct SSDPrefillArguments<'a, B: Backend> {- pub x: &'a B::NativeBuffer,- pub dt: &'a B::NativeBuffer, // raw dt values- pub b: &'a B::NativeBuffer,- pub c: &'a B::NativeBuffer,- pub d: &'a B::NativeBuffer,- pub z: &'a B::NativeBuffer,- pub state: &'a mut B::NativeBuffer,- pub y: &'a mut B::NativeBuffer,+ pub x: &'a B::Buffer,+ pub dt: &'a B::Buffer, // raw dt values+ pub b: &'a B::Buffer,+ pub c: &'a B::Buffer,+ pub d: &'a B::Buffer,+ pub z: &'a B::Buffer,+ pub state: &'a mut B::Buffer,+ pub y: &'a mut B::Buffer,pub suffix_len: usize,pub group_size: i32,pub state_size: i32,@@ -55,7 +55,7 @@ impl<B: Backend> SSDPrefillKernels<B> {pub fn encode(&self,- compute_encoder: &mut B::ComputeEncoder,+ compute_encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,args: SSDPrefillArguments<B>,mode: SSDPrefillMode,) {diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rsindex 9b2caf29a..b98c4bdfa 100644--- a/crates/uzu/src/backends/common/mod.rs+++ b/crates/uzu/src/backends/common/mod.rs@@ -1,22 +1,17 @@-mod allocator;mod backend;mod buffer;-mod buffer_lifetime;mod command_buffer;mod context;-mod copy_encoder;mod event;pub mod gpu_types;pub mod kernel;-mod native_buffer;-pub use allocator::{AllocError, Allocator};pub use backend::Backend;pub use buffer::Buffer;-pub use buffer_lifetime::BufferLifetime;-pub use command_buffer::CommandBuffer;+pub use command_buffer::{+ CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending,+};pub use context::{Context, DeviceClass};-pub use copy_encoder::CopyEncoder;pub use event::Event;pub use kernel::Kernels;-pub use native_buffer::NativeBuffer;diff --git a/crates/uzu/src/backends/common/native_buffer.rs b/crates/uzu/src/backends/common/native_buffer.rsdeleted file mode 100644index b5cb6c798..000000000--- a/crates/uzu/src/backends/common/native_buffer.rs+++ /dev/null@@ -1,18 +0,0 @@-use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};--use super::Backend;--pub trait NativeBuffer: Debug {- type Backend: Backend;-- fn set_label(- &mut self,- label: Option<&str>,- );-- fn cpu_ptr(&self) -> NonNull<c_void>;-- fn length(&self) -> usize;-- fn id(&self) -> usize;-}diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rsindex 6ac60bea7..d6a2a5a6e 100644--- a/crates/uzu/src/backends/cpu/backend.rs+++ b/crates/uzu/src/backends/cpu/backend.rs@@ -8,10 +8,8 @@ pub struct Cpu;impl Backend for Cpu {type Context = CpuContext;- type NativeBuffer = Box<[u8]>;type CommandBuffer = CpuCommandBuffer;- type ComputeEncoder = CpuCommandBuffer;- type CopyEncoder = CpuCommandBuffer;+ type Buffer = Box<[u8]>;type Event = RefCell<u64>;type Kernels = CpuKernels;type Error = CpuError;diff --git a/crates/uzu/src/backends/cpu/native_buffer.rs b/crates/uzu/src/backends/cpu/buffer.rssimilarity index 84%rename from crates/uzu/src/backends/cpu/native_buffer.rsrename to crates/uzu/src/backends/cpu/buffer.rsindex a885455be..306bc05e5 100644--- a/crates/uzu/src/backends/cpu/native_buffer.rs+++ b/crates/uzu/src/backends/cpu/buffer.rs@@ -1,9 +1,9 @@use std::{os::raw::c_void, ptr::NonNull};use super::Cpu;-use crate::backends::common::NativeBuffer;+use crate::backends::common::Buffer;-impl NativeBuffer for Box<[u8]> {+impl Buffer for Box<[u8]> {type Backend = Cpu;fn set_label(diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rsindex 600e880d6..726f09ff6 100644--- a/crates/uzu/src/backends/cpu/command_buffer.rs+++ b/crates/uzu/src/backends/cpu/command_buffer.rs@@ -5,16 +5,29 @@ use std::{use super::Cpu;use crate::backends::{- common::{CommandBuffer, CopyEncoder},+ common::{+ CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending,+ },cpu::error::CpuError,};pub struct CpuCommandBuffer {commands: Vec<Box<dyn Fn()>>,- completion_handlers: Vec<Box<dyn FnOnce(Result<(), CpuError>)>>,+ completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBuffer, CpuError>)>>,gpu_execution_time_ms: OnceCell<f64>,}+impl CommandBuffer for CpuCommandBuffer {+ type Backend = Cpu;++ type Initial = CpuCommandBuffer;+ type Encoding = CpuCommandBuffer;+ type Executable = CpuCommandBuffer;+ type Pending = CpuCommandBuffer;+ type Completed = CpuCommandBuffer;+}+impl CpuCommandBuffer {pub fn new() -> CpuCommandBuffer {CpuCommandBuffer {@@ -32,21 +45,41 @@ impl CpuCommandBuffer {}}-impl CommandBuffer for CpuCommandBuffer {- type Backend = Cpu;+impl CommandBufferInitial for CpuCommandBuffer {+ type CommandBuffer = CpuCommandBuffer;++ fn start_encoding(self) -> CpuCommandBuffer {+ self+ }+}- fn with_compute_encoder<T>(+impl CommandBufferEncoding for CpuCommandBuffer {+ type CommandBuffer = CpuCommandBuffer;++ fn encode_copy(&mut self,- callback: impl FnOnce(&mut CpuCommandBuffer) -> T,- ) -> T {- callback(self)+ src: &Box<[u8]>,+ dst: &mut Box<[u8]>,+ size: usize,+ ) {+ let src = src.as_ptr();+ let dst = dst.as_ptr() as *mut u8;+ self.push_command(Box::new(move || unsafe {+ std::ptr::copy(src, dst, size);+ }))}- fn with_copy_encoder<T>(+ fn encode_fill(&mut self,- callback: impl FnOnce(&mut CpuCommandBuffer) -> T,- ) -> T {- callback(self)+ dst: &mut Box<[u8]>,+ range: std::ops::Range<usize>,+ value: u8,+ ) {+ let size = range.end - range.start;+ let dst = dst[range].as_ptr() as *mut u8;+ self.push_command(Box::new(move || unsafe {+ dst.write_bytes(value, size);+ }))}fn encode_wait_for_event(@@ -69,59 +102,53 @@ impl CommandBuffer for CpuCommandBuffer {fn add_completion_handler(&mut self,- handler: impl FnOnce(Result<(), CpuError>) + 'static,+ handler: impl FnOnce(Result<&CpuCommandBuffer, CpuError>) + 'static,) {self.completion_handlers.push(Box::new(handler))}- fn submit(&mut self) {+ fn end_encoding(self) -> CpuCommandBuffer {+ self+ }+}++impl CommandBufferExecutable for CpuCommandBuffer {+ type CommandBuffer = CpuCommandBuffer;++ fn submit(mut self) -> CpuCommandBuffer {let start = Instant::now();for command in &self.commands {command()}+self.gpu_execution_time_ms.set((Instant::now() - start).as_secs_f64() * 1000.0).expect("gpu execution time already set");- for completion_handler in self.completion_handlers.drain(..) {- completion_handler(Ok(()));- }- }- fn wait_until_completed(&self) -> Result<(), CpuError> {- Ok(())- }+ for completion_handler in self.completion_handlers.drain(..).collect::<Vec<_>>() {+ completion_handler(Ok(&self));+ }- fn gpu_execution_time_ms(&self) -> Option<f64> {- self.gpu_execution_time_ms.get().copied()+ CpuCommandBuffer {+ commands: Vec::new(),+ completion_handlers: Vec::new(),+ gpu_execution_time_ms: self.gpu_execution_time_ms,+ }}}-impl CopyEncoder for CpuCommandBuffer {- type Backend = Cpu;+impl CommandBufferPending for CpuCommandBuffer {+ type CommandBuffer = CpuCommandBuffer;- fn encode_copy(- &mut self,- src: &Box<[u8]>,- dst: &mut Box<[u8]>,- size: usize,- ) {- let src = src.as_ptr();- let dst = dst.as_ptr() as *mut u8;- self.push_command(Box::new(move || unsafe {- std::ptr::copy(src, dst, size);- }))+ fn wait_until_completed(self) -> Result<CpuCommandBuffer, CpuError> {+ Ok(self)}+}- fn encode_fill(- &mut self,- dst: &mut Box<[u8]>,- range: std::ops::Range<usize>,- value: u8,- ) {- let size = range.end - range.start;- let dst = dst[range].as_ptr() as *mut u8;- self.push_command(Box::new(move || unsafe {- dst.write_bytes(value, size);- }))+impl CommandBufferCompleted for CpuCommandBuffer {+ type CommandBuffer = CpuCommandBuffer;++ fn gpu_execution_time_ms(&self) -> Option<f64> {+ self.gpu_execution_time_ms.get().copied()}}diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rsindex 4efd26ec3..79c450bfc 100644--- a/crates/uzu/src/backends/cpu/context.rs+++ b/crates/uzu/src/backends/cpu/context.rs@@ -1,7 +1,7 @@use std::{cell::RefCell, rc::Rc};use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};-use crate::backends::common::{Allocator, Context, DeviceClass};+use crate::backends::common::{Context, DeviceClass};pub struct CpuContext;@@ -20,10 +20,6 @@ impl Context for CpuContext {false}- fn allocator(&self) -> &Allocator<Self::Backend> {- unimplemented!()- }-fn create_buffer(&self,size: usize,diff --git a/crates/uzu/src/backends/cpu/mod.rs b/crates/uzu/src/backends/cpu/mod.rsindex bd15afca5..97a2b7485 100644--- a/crates/uzu/src/backends/cpu/mod.rs+++ b/crates/uzu/src/backends/cpu/mod.rs@@ -1,10 +1,10 @@mod backend;+mod buffer;mod command_buffer;mod context;mod error;mod event;mod kernel;-mod native_buffer;pub use backend::Cpu;diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex c8e37bb01..97b583bda 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -1,7 +1,7 @@-use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder, MTLEvent};+use metal::{MTLBuffer, MTLEvent};use objc2::{rc::Retained, runtime::ProtocolObject};-use super::{context::MetalContext, error::MetalError, kernel::dsl::MetalKernels};+use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::dsl::MetalKernels};use crate::backends::common::Backend;#[derive(Debug, Clone)]@@ -9,10 +9,8 @@ pub struct Metal;impl Backend for Metal {type Context = MetalContext;- type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;- type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;- type ComputeEncoder = Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>;- type CopyEncoder = Retained<ProtocolObject<dyn MTLBlitCommandEncoder>>;+ type CommandBuffer = MetalCommandBuffer;+ type Buffer = Retained<ProtocolObject<dyn MTLBuffer>>;type Event = Retained<ProtocolObject<dyn MTLEvent>>;type Kernels = MetalKernels;type Error = MetalError;diff --git a/crates/uzu/src/backends/metal/native_buffer.rs b/crates/uzu/src/backends/metal/buffer.rssimilarity index 82%rename from crates/uzu/src/backends/metal/native_buffer.rsrename to crates/uzu/src/backends/metal/buffer.rsindex c36646fe5..84c0a4914 100644--- a/crates/uzu/src/backends/metal/native_buffer.rs+++ b/crates/uzu/src/backends/metal/buffer.rs@@ -4,9 +4,9 @@ use metal::{MTLBuffer, MTLResourceExt};use objc2::{rc::Retained, runtime::ProtocolObject};use super::Metal;-use crate::backends::common::NativeBuffer;+use crate::backends::common::Buffer;-impl NativeBuffer for Retained<ProtocolObject<dyn MTLBuffer>> {+impl Buffer for Retained<ProtocolObject<dyn MTLBuffer>> {type Backend = Metal;fn set_label(diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex 8c7608cd8..8b72aaf00 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -1,12 +1,31 @@-use std::{cell::Cell, ops::Deref};+use std::cell::Cell;use metal::{- MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandBufferStatus, MTLCommandEncoder, MTLEvent,+ MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,+ MTLCommandBufferHandler, MTLCommandBufferStatus, MTLCommandEncoder, MTLComputeCommandEncoder, MTLEvent,};-use objc2::{rc::Retained, runtime::ProtocolObject};+use objc2::{Message, rc::Retained, runtime::ProtocolObject};use super::Metal;-use crate::backends::{common::CommandBuffer, metal::error::MetalError};+use crate::backends::{+ common::{+ CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending,+ },+ metal::error::MetalError,+};++pub struct MetalCommandBuffer;++impl CommandBuffer for MetalCommandBuffer {+ type Backend = Metal;++ type Initial = MetalCommandBufferInitial;+ type Encoding = MetalCommandBufferEncoding;+ type Executable = MetalCommandBufferExecutable;+ type Pending = MetalCommandBufferPending;+ type Completed = MetalCommandBufferCompleted;+}fn command_buffer_result(command_buffer: &ProtocolObject<dyn MTLCommandBuffer>) -> Result<(), MetalError> {match (command_buffer.status(), command_buffer.error()) {@@ -16,33 +35,110 @@ fn command_buffer_result(command_buffer: &ProtocolObject<dyn MTLCommandBuffer>)}}-impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {- type Backend = Metal;+pub struct MetalCommandBufferInitial {+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,+}- fn with_compute_encoder<T>(- &mut self,- callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,- ) -> T {- let mut encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");+impl MetalCommandBufferInitial {+ pub fn new(command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>) -> Self {+ Self {+ command_buffer,+ }+ }+}++impl CommandBufferInitial for MetalCommandBufferInitial {+ type CommandBuffer = MetalCommandBuffer;++ fn start_encoding(self) -> MetalCommandBufferEncoding {+ MetalCommandBufferEncoding {+ command_buffer: self.command_buffer,+ encoding_state: MetalCommandBufferEncodingEncodingState::None,+ }+ }+}++enum MetalCommandBufferEncodingEncodingState {+ None,+ Compute(Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>),+ Blit(Retained<ProtocolObject<dyn MTLBlitCommandEncoder>>),+}++pub struct MetalCommandBufferEncoding {+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ encoding_state: MetalCommandBufferEncodingEncodingState,+}++impl MetalCommandBufferEncoding {+ fn ensure_none(&mut self) {+ match &mut self.encoding_state {+ MetalCommandBufferEncodingEncodingState::None => return,+ MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) => compute_encoder.end_encoding(),+ MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) => blit_encoder.end_encoding(),+ };++ self.encoding_state = MetalCommandBufferEncodingEncodingState::None;+ }++ pub(super) fn ensure_compute(&mut self) -> &mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {+ if !matches!(self.encoding_state, MetalCommandBufferEncodingEncodingState::Compute(_)) {+ self.ensure_none();+ self.encoding_state = MetalCommandBufferEncodingEncodingState::Compute(+ self.command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder"),+ );+ }++ let MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) = &mut self.encoding_state else {+ unreachable!()+ };+ compute_encoder+ }- let ret = callback(&mut encoder);+ fn ensure_blit(&mut self) -> &mut Retained<ProtocolObject<dyn MTLBlitCommandEncoder>> {+ if !matches!(self.encoding_state, MetalCommandBufferEncodingEncodingState::Blit(_)) {+ self.ensure_none();+ self.encoding_state = MetalCommandBufferEncodingEncodingState::Blit(+ self.command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder"),+ );+ }- encoder.end_encoding();+ let MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) = &mut self.encoding_state else {+ unreachable!()+ };+ blit_encoder+ }+}- ret+impl Drop for MetalCommandBufferEncoding {+ fn drop(&mut self) {+ self.ensure_none();}+}- fn with_copy_encoder<T>(+impl CommandBufferEncoding for MetalCommandBufferEncoding {+ type CommandBuffer = MetalCommandBuffer;++ fn encode_copy(&mut self,- callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,- ) -> T {- let mut encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");+ src: &Retained<ProtocolObject<dyn MTLBuffer>>,+ dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,+ size: usize,+ ) {+ assert!(src.length() >= size && dst.length() >= size);- let ret = callback(&mut encoder);+ self.ensure_blit().copy_buffer_to_buffer(src, 0, dst, 0, size);+ }- encoder.end_encoding();+ fn encode_fill(+ &mut self,+ dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,+ range: std::ops::Range<usize>,+ value: u8,+ ) {+ assert!(range.end > range.start && range.end <= dst.length());+ assert!(range.start % 4 == 0 && range.end % 4 == 0);- ret+ self.ensure_blit().fill_buffer_range_value(dst, range, value);}fn encode_wait_for_event(@@ -50,7 +146,8 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {event: &Retained<ProtocolObject<dyn MTLEvent>>,value: u64,) {- self.encode_wait_for_event_value(event, value);+ self.ensure_none();+ self.command_buffer.encode_wait_for_event_value(event, value);}fn encode_signal_event(@@ -58,30 +155,75 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {event: &Retained<ProtocolObject<dyn MTLEvent>>,value: u64,) {- self.encode_signal_event_value(event, value);+ self.ensure_none();+ self.command_buffer.encode_signal_event_value(event, value);}fn add_completion_handler(&mut self,- handler: impl FnOnce(Result<(), MetalError>) + 'static,+ handler: impl FnOnce(Result<&MetalCommandBufferCompleted, MetalError>) + 'static,) {let cell = Cell::new(Some(handler));- self.deref().add_completed_handler(&MTLCommandBufferHandler::new(move |command_buffer| {- cell.take().expect("completion handler called more than once")(command_buffer_result(command_buffer))+ self.command_buffer.add_completed_handler(&MTLCommandBufferHandler::new(move |command_buffer| {+ let cbuf_ref = MetalCommandBufferCompleted {+ command_buffer: command_buffer.retain(),+ };+ cell.take().expect("completion handler called more than once")(+ command_buffer_result(command_buffer).map(|_| &cbuf_ref),+ )}));}- fn submit(&mut self) {- self.commit();+ fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {+ self.ensure_none();++ MetalCommandBufferExecutable {+ command_buffer: self.command_buffer.clone(),+ }+ }+}++pub struct MetalCommandBufferExecutable {+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,+}++impl CommandBufferExecutable for MetalCommandBufferExecutable {+ type CommandBuffer = MetalCommandBuffer;++ fn submit(self) -> MetalCommandBufferPending {+ self.command_buffer.commit();++ MetalCommandBufferPending {+ command_buffer: self.command_buffer,+ }}+}++pub struct MetalCommandBufferPending {+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,+}++impl CommandBufferPending for MetalCommandBufferPending {+ type CommandBuffer = MetalCommandBuffer;- fn wait_until_completed(&self) -> Result<(), MetalError> {- self.deref().wait_until_completed();- command_buffer_result(self)+ fn wait_until_completed(self) -> Result<MetalCommandBufferCompleted, MetalError> {+ self.command_buffer.wait_until_completed();++ command_buffer_result(&self.command_buffer).map(|_| MetalCommandBufferCompleted {+ command_buffer: self.command_buffer,+ })}+}++pub struct MetalCommandBufferCompleted {+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,+}++impl CommandBufferCompleted for MetalCommandBufferCompleted {+ type CommandBuffer = MetalCommandBuffer;fn gpu_execution_time_ms(&self) -> Option<f64> {- match (self.kernel_start_time(), self.kernel_end_time()) {+ match (self.command_buffer.kernel_start_time(), self.command_buffer.kernel_end_time()) {(Some(start), Some(end)) => Some((end - start) * 1000.0),_ => None,}diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex 89973e2a0..69e10c1ed 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -1,14 +1,17 @@use std::{cell::RefCell, collections::HashMap, env, rc::Rc};use metal::{- MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandBuffer, MTLCommandQueue,- MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,- MTLLibrary, MTLResourceOptions,+ MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue, MTLCommandQueueExt,+ MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues, MTLLibrary,+ MTLResourceOptions,};use objc2::{rc::Retained, runtime::ProtocolObject};use super::{Metal, error::MetalError, kernel, metal_extensions::LibraryPipelineExtensions};-use crate::backends::common::{Allocator, Context, DeviceClass as CommonDeviceClass};+use crate::backends::{+ common::{Context, DeviceClass as CommonDeviceClass},+ metal::command_buffer::MetalCommandBufferInitial,+};/// Apple GPU architecture generation./// Based on Apple GPU family naming convention (e.g., "applegpu_g13p").@@ -127,7 +130,6 @@ pub struct MetalContext {architecture: DeviceArchitecture,library: Retained<ProtocolObject<dyn MTLLibrary>>,pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,- allocator: Allocator<Metal>,}impl MetalContext {@@ -190,13 +192,12 @@ impl Context for MetalContext {let architecture = DeviceArchitecture::from_device(&device);- Ok(Rc::new_cyclic(|weak_self| Self {+ Ok(Rc::new(Self {device,command_queue,architecture,library,pipeline_cache: RefCell::new(HashMap::new()),- allocator: Allocator::new(weak_self.clone()),}))}@@ -221,10 +222,6 @@ impl Context for MetalContext {matches!(upper.as_str(), "1" | "YES" | "TRUE")}- fn allocator(&self) -> &Allocator<Metal> {- &self.allocator- }-fn create_buffer(&self,size: usize,@@ -232,8 +229,10 @@ impl Context for MetalContext {self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).ok_or(MetalError::CannotCreateBuffer)}- fn create_command_buffer(&self) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MetalError> {- self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)+ fn create_command_buffer(&self) -> Result<MetalCommandBufferInitial, MetalError> {+ Ok(MetalCommandBufferInitial::new(+ self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,+ ))}fn create_event(&self) -> Result<Retained<ProtocolObject<dyn MTLEvent>>, MetalError> {diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rsdeleted file mode 100644index e74776e15..000000000--- a/crates/uzu/src/backends/metal/copy_encoder.rs+++ /dev/null@@ -1,34 +0,0 @@-use std::ops::Range;--use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::Metal;-use crate::backends::common::{Backend, CopyEncoder};--impl CopyEncoder for Retained<ProtocolObject<dyn MTLBlitCommandEncoder>> {- type Backend = Metal;-- fn encode_copy(- &mut self,- src: &<Self::Backend as Backend>::NativeBuffer,- dst: &mut <Self::Backend as Backend>::NativeBuffer,- size: usize,- ) {- assert!(src.length() >= size && dst.length() >= size);-- self.copy_buffer_to_buffer(src, 0, dst, 0, size);- }-- fn encode_fill(- &mut self,- dst: &mut <Self::Backend as Backend>::NativeBuffer,- range: Range<usize>,- value: u8,- ) {- assert!(range.end > range.start && range.end <= dst.length());- assert!(range.start % 4 == 0 && range.end % 4 == 0);-- self.fill_buffer_range_value(dst, range, value);- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex d9482d193..f39fd382c 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -6,9 +6,12 @@ pub use dispatch_descriptor::choose_dispatch_descriptor;use crate::{DataType,backends::{- common::kernel::matmul::{- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulArguments,- MatmulError, MatmulKernel,+ common::{+ Backend, CommandBuffer,+ kernel::matmul::{+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,+ MatmulArguments, MatmulError, MatmulKernel,+ },},metal::{Metal, context::MetalContext},},@@ -29,7 +32,7 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {fn encode(&mut self,context: &MetalContext,- encoder: &mut <Metal as crate::backends::common::Backend>::ComputeEncoder,+ encoder: &mut <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding,arguments: FullPrecisionMatmulArguments<Metal>,) {let mut matmul_arguments = MatmulArguments {diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rsdeleted file mode 100644index 9c6e5cf95..000000000--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs+++ /dev/null@@ -1,84 +0,0 @@-use metal::{MTLBuffer, MTLCompareFunction, MTLComputeCommandEncoder};-use objc2::{msg_send, rc::Retained, runtime::ProtocolObject};--/// Low-level, unsafe conditional control of Metal encoders.-/// This is internal; users should prefer the safe `ComputeEncoderConditional::condition`.-trait ComputeEncoderRawConditional {- fn encode_start_if(- &self,- predicate: &ProtocolObject<dyn MTLBuffer>,- offset: usize,- comparison: MTLCompareFunction,- reference_value: u32,- );-- fn encode_start_else(&self);-- fn encode_end_if(&self) -> bool;-}--/// Safe conditional wrapper that uses the raw unsafe methods internally.-pub trait ComputeEncoderConditional {- fn condition<IfBlock, ElseBlock>(- &mut self,- predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,- offset: usize,- if_block: IfBlock,- else_block: Option<ElseBlock>,- ) where- IfBlock: FnOnce(&mut Self),- ElseBlock: FnOnce(&mut Self);-}--impl ComputeEncoderRawConditional for Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {- fn encode_start_if(- &self,- predicate: &ProtocolObject<dyn MTLBuffer>,- offset: usize,- comparison: MTLCompareFunction,- reference_value: u32,- ) {- let _: () = unsafe {- msg_send![- self,- encodeStartIf: predicate,- offset: offset,- comparison: comparison,- referenceValue: reference_value- ]- };- }-- fn encode_start_else(&self) {- let _: () = unsafe { msg_send![self, encodeStartElse] };- }-- fn encode_end_if(&self) -> bool {- let result: bool = unsafe { msg_send![self, encodeEndIf] };- result- }-}--impl<T> ComputeEncoderConditional for T-where- T: ComputeEncoderRawConditional,-{- fn condition<IfBlock, ElseBlock>(- &mut self,- predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,- offset: usize,- if_block: IfBlock,- else_block: Option<ElseBlock>,- ) where- IfBlock: FnOnce(&mut Self),- ElseBlock: FnOnce(&mut Self),- {- self.encode_start_if(predicate, offset, MTLCompareFunction::Equal, 0);- if_block(self);- if let Some(else_block) = else_block {- self.encode_start_else();- else_block(self);- }- self.encode_end_if();- }-}diff --git a/crates/uzu/src/backends/metal/metal_extensions/mod.rs b/crates/uzu/src/backends/metal/metal_extensions/mod.rsindex fd8b90ecb..963d89656 100644--- a/crates/uzu/src/backends/metal/metal_extensions/mod.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/mod.rs@@ -1,10 +1,8 @@-mod compute_command_encoder_extensions_conditional;mod compute_command_encoder_extensions_set_value;mod data_type;mod function_constant_values_extensions_set_value;mod library_extensions_pipeline;-pub use compute_command_encoder_extensions_conditional::ComputeEncoderConditional;pub use compute_command_encoder_extensions_set_value::ComputeEncoderSetValue;pub use data_type::MetalDataTypeExt;pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 64ed3aecb..2a7ef3c41 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -1,12 +1,11 @@mod backend;+mod buffer;mod command_buffer;mod context;-mod copy_encoder;mod error;mod event;mod kernel;mod metal_extensions;-mod native_buffer;pub use backend::Metal;pub use kernel::matmul::choose_dispatch_descriptor;diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 18dcfdf47..6c5bf3e02 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -9,7 +9,7 @@ use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{DataType,- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,@@ -111,9 +111,7 @@ impl<B: Backend> Classifier<B> {num_labels,);- self.context.reset_command_buffer();-- let encoding_params = EncodingParameters::new(false, true, false);+ let encoding_params = EncodingParameters::new();self.context.embed@@ -162,8 +160,12 @@ impl<B: Backend> Classifier<B> {.encode(&mut state, &encoding_params, &mut self.context.command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context.command_buffer.submit();- self.context.command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ self.context+ .replace_command_buffer()+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;let logits = self.copy_logits_from_state(&state)?;@@ -191,9 +193,7 @@ impl<B: Backend> Classifier<B> {num_labels,);- self.context.reset_command_buffer();-- let encoding_params = EncodingParameters::new(false, true, false);+ let encoding_params = EncodingParameters::new();self.context.embed@@ -221,8 +221,12 @@ impl<B: Backend> Classifier<B> {.encode(&mut state, &encoding_params, &mut self.context.command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context.command_buffer.submit();- self.context.command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ self.context+ .replace_command_buffer()+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;let logits = self.copy_logits_from_state(&state)?;diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rsindex ef05c7ef3..4e73fa3c7 100644--- a/crates/uzu/src/classifier/classifier_context.rs+++ b/crates/uzu/src/classifier/classifier_context.rs@@ -2,7 +2,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};use crate::{DataType,- backends::common::{Backend, Context, Kernels, kernel::SigmoidKernel},+ backends::common::{Backend, CommandBuffer, CommandBufferInitial, Context, Kernels, kernel::SigmoidKernel},classifier::ClassifierError,config::{ClassifierModelConfig, ModelMetadata},encodable_block::{@@ -20,7 +20,7 @@ use crate::{pub struct ClassifierContext<B: Backend> {pub context: Rc<B::Context>,- pub command_buffer: B::CommandBuffer,+ pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,pub scratch_buffers: ScratchBuffers<B>,@@ -45,7 +45,8 @@ impl<B: Backend> ClassifierContext<B> {pub fn new(model_path: &Path) -> Result<Self, Error> {let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;- let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?;+ let command_buffer =+ context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?.start_encoding();let config_path = model_path.join("config.json");if !config_path.exists() {@@ -310,7 +311,10 @@ impl<B: Backend> ClassifierContext<B> {Ok(Rc::new(rotation))}- pub fn reset_command_buffer(&mut self) {- self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");+ pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {+ std::mem::replace(+ &mut self.command_buffer,+ self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),+ )}}diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rsindex f0fab5350..11caea45b 100644--- a/crates/uzu/src/encodable_block/activation.rs+++ b/crates/uzu/src/encodable_block/activation.rs@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{ActivationKernel, Kernels},},config::Activation as ActivationConfig,@@ -40,16 +40,12 @@ impl<B: Backend> Activation<B> {}impl<B: Backend> EncodableBlock<B> for Activation<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let input_array = arrays[0].borrow();let output_array = arrays[1].borrow_mut();@@ -70,7 +66,8 @@ impl<B: Backend> EncodableBlock<B> for Activation<B> {output_array.buffer().borrow_mut().deref_mut(),n as u32,act_type,- encoder,+ command_buffer,);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex bf8f729fa..1a8bf6bf6 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -139,28 +139,9 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));-- if parameters.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;- }- Ok(())- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- compute_encoder: &mut B::ComputeEncoder,- ) {let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);let qkv_array = qkv_binding[0].borrow();@@ -258,7 +239,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {// For classifiers (no KV cache): extract values from QKV into a dedicated extracted_values buffer.if !has_kv_cache {self.update_kv_cache_inplace_kernel.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,qkv_buf_borrow.deref(),// keys already in desired layout; harmless overwriterotated_keys_buf_borrow.deref_mut(),@@ -269,7 +250,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {suffix_length as u32,0u32,max_sequence_length as u32,- compute_encoder,+ command_buffer,);}@@ -293,7 +274,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {};let attention_bias_buf_rc = attention_bias_array_borrow.as_ref().map(|a| a.buffer());let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());- let attention_bias_buffer: Option<&B::NativeBuffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());+ let attention_bias_buffer: Option<&B::Buffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);let sums_binding = state.arrays(&[ArrayId::AttentionSums]);@@ -314,7 +295,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {let sinks_borrow = sinks_binding.as_ref().map(|binding| binding[0].borrow());let sinks_buf_rc = sinks_borrow.as_ref().map(|b| b.buffer());let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());- let sinks_buffer: Option<&B::NativeBuffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());+ let sinks_buffer: Option<&B::Buffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());// Only update KV cache for LLM mode (not for classifiers)if has_kv_cache {@@ -329,16 +310,16 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {suffix_length as u32,segment_prefix_length as u32,max_sequence_length as u32,- compute_encoder,+ command_buffer,);}- let key_cache_buffer: &B::NativeBuffer = if has_kv_cache {+ let key_cache_buffer: &B::Buffer = if has_kv_cache {key_cache_buf_borrow.as_ref().unwrap().deref()} else {rotated_keys_buf_borrow.deref()};- let value_cache_buffer: &B::NativeBuffer = if has_kv_cache {+ let value_cache_buffer: &B::Buffer = if has_kv_cache {value_cache_buf_borrow.as_ref().unwrap().deref()} else {extracted_values_buf_borrow.as_ref().unwrap().deref()@@ -383,7 +364,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {scale,};self.gemm_block- .encode(state.context(), compute_encoder, args)+ .encode(state.context(), command_buffer, args).expect("Failed to encode AttentionGemmBlock");},KernelVariant::SinglePass => {@@ -410,7 +391,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {sinks_buffer,num_heads as u32,suffix_length as u32,- compute_encoder,+ command_buffer,)},KernelVariant::TwoPass => {@@ -443,7 +424,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {mask_q_seq_stride_opt,mask_head_stride_opt,sinks_buffer,- compute_encoder,+ command_buffer,);kernel_pass2.encode(partials_buf_borrow.deref(),@@ -452,10 +433,12 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {attention_output_buf_borrow.deref_mut(),num_heads as u32,suffix_length as u32,- compute_encoder,+ command_buffer,);},}++ Ok(())}}diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex 1c5637417..33b3ca455 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -2,11 +2,9 @@ use std::rc::Rc;use objc2::rc::autoreleasepool;-#[cfg(not(feature = "tracing"))]-use crate::backends::common::CommandBuffer;use crate::{DataType,- backends::common::Backend,+ backends::common::{Backend, CommandBuffer},config::TransformerLayerConfig,encodable_block::{Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,@@ -249,18 +247,9 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- #[cfg(not(feature = "tracing"))]- {- if self.supports_shared_encoder() {- command_buffer- .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));- return Ok(());- }- }-#[cfg(feature = "tracing")]let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();@@ -353,72 +342,4 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {let _ = parameters;Ok(())}-- fn supports_shared_encoder(&self) -> bool {- #[cfg(feature = "tracing")]- {- false- }-- #[cfg(not(feature = "tracing"))]- {- self.copy_main_to_shortcut_mixer.supports_shared_encoder()- && self.pre_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)- && self.qkv_projection.supports_shared_encoder()- && self.qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)- && self.rope.supports_shared_encoder()- && self.attention.supports_shared_encoder()- && self.out_projection.supports_shared_encoder()- && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)- && self.mixer_residual_add.supports_shared_encoder()- && self.copy_main_to_shortcut_mlp.supports_shared_encoder()- && self.pre_mlp_norm.supports_shared_encoder()- && self.mlp.supports_shared_encoder()- && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)- && self.mlp_residual_add.supports_shared_encoder()- }- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {- debug_assert!(- self.supports_shared_encoder(),- "encode_with_shared_encoder called on unsupported ClassifierLayer"- );-- self.copy_main_to_shortcut_mixer.encode_with_shared_encoder(state, parameters, encoder);-- if let Some(ref pre_attn_norm) = self.pre_attention_norm {- pre_attn_norm.encode_with_shared_encoder(state, parameters, encoder);- }-- self.qkv_projection.encode_with_shared_encoder(state, parameters, encoder);- if let Some(ref qk_norm) = self.qk_norm {- qk_norm.encode_with_shared_encoder(state, parameters, encoder);- }- self.rope.encode_with_shared_encoder(state, parameters, encoder);- self.attention.encode_with_shared_encoder(state, parameters, encoder);- self.out_projection.encode_with_shared_encoder(state, parameters, encoder);-- if let Some(ref post_attn_norm) = self.post_attention_norm {- post_attn_norm.encode_with_shared_encoder(state, parameters, encoder);- }-- self.mixer_residual_add.encode_with_shared_encoder(state, parameters, encoder);-- self.copy_main_to_shortcut_mlp.encode_with_shared_encoder(state, parameters, encoder);-- self.pre_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);- self.mlp.encode_with_shared_encoder(state, parameters, encoder);-- if let Some(ref post_mlp_norm) = self.post_mlp_norm {- post_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);- }-- self.mlp_residual_add.encode_with_shared_encoder(state, parameters, encoder);- }}diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex 56f88326f..75b4338c7 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -4,7 +4,7 @@ use std::rc::Rc;use crate::{DataType, DecoderConfig,- backends::common::Backend,+ backends::common::{Backend, CommandBuffer},config::{DecoderLayerType, MixerConfig},encodable_block::{EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,@@ -162,20 +162,11 @@ impl<B: Backend> Decoder<B> {}impl<B: Backend> EncodableBlock<B> for Decoder<B> {- fn encode_with_shared_encoder(- &self,- _state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- _encoder: &mut B::ComputeEncoder,- ) {- unimplemented!("Decoder does not support shared encoder")- }-fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {self.embed.encode(state, parameters, command_buffer)?;diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rsindex 8b1e0a85a..c9e0fc151 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{FullPrecisionEmbeddingLookupKernel, Kernels},},forward_pass::state::{ArrayId, ForwardPassState},@@ -42,7 +42,7 @@ pub enum FullPrecisionEmbeddingLookupError<B: Backend> {pub struct FullPrecisionEmbeddingLookup<B: Backend> {kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,- weights_buffer: Rc<RefCell<B::NativeBuffer>>,+ weights_buffer: Rc<RefCell<B::Buffer>>,vocab_size: u32,model_dim: u32,input_scale: f32,@@ -91,16 +91,12 @@ impl<B: Backend> FullPrecisionEmbeddingLookup<B> {}impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);let batch_size = state.active_suffix_length();let token_ids_array = arrays[0].borrow_mut();@@ -114,7 +110,8 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {self.vocab_size,self.model_dim,self.input_scale,- encoder,+ command_buffer,);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rsindex 2af5bc31b..d993926f0 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs@@ -10,7 +10,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},},forward_pass::state::{ArrayId, ForwardPassState},@@ -42,7 +42,7 @@ pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {pub struct FullPrecisionEmbeddingReadout<B: Backend> {kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,- weights_buffer: Rc<RefCell<B::NativeBuffer>>,+ weights_buffer: Rc<RefCell<B::Buffer>>,vocab_size: usize,model_dim: usize,}@@ -94,20 +94,16 @@ impl<B: Backend> FullPrecisionEmbeddingReadout<B> {}impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);let batch_size = state.sampling_length();if batch_size == 0 {- return;+ return Ok(());}let sampling_start = state.sampling_start();@@ -117,7 +113,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {self.kernel.borrow_mut().encode(state.context(),- encoder,+ command_buffer,FullPrecisionMatmulArguments {a: input_array.buffer().borrow().deref(),a_offset: sampling_start * self.model_dim * element_size,@@ -129,5 +125,6 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {output_dim: self.vocab_size,},);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rsindex 9c5daed29..bde5e27ff 100644--- a/crates/uzu/src/encodable_block/full_precision_linear.rs+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs@@ -10,7 +10,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},},forward_pass::state::{ArrayId, ForwardPassState},@@ -50,8 +50,8 @@ pub enum FullPrecisionLinearError<B: Backend> {pub struct FullPrecisionLinear<B: Backend> {kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,- bias_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,- weights_buffer: Rc<RefCell<B::NativeBuffer>>,+ bias_buffer: Option<Rc<RefCell<B::Buffer>>>,+ weights_buffer: Rc<RefCell<B::Buffer>>,input_dim: usize,output_dim: usize,input_array_id: ArrayId,@@ -126,16 +126,12 @@ impl<B: Backend> FullPrecisionLinear<B> {}impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();let input_array = arrays[0].borrow_mut();@@ -144,7 +140,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());self.kernel.borrow_mut().encode(state.context(),- encoder,+ command_buffer,FullPrecisionMatmulArguments {a: input_array.buffer().borrow().deref(),a_offset: 0,@@ -156,5 +152,6 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {output_dim: self.output_dim,},);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 25148726c..b51a5d85c 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -5,11 +5,9 @@ use std::rc::Rc;use objc2::rc::autoreleasepool;use super::MixerExecutables;-#[cfg(not(feature = "tracing"))]-use crate::backends::common::CommandBuffer;use crate::{DataType, DecoderLayerConfig,- backends::common::Backend,+ backends::common::{Backend, CommandBuffer},config::{DecoderLayerType, MixerConfig},encodable_block::{Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,@@ -260,20 +258,9 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- // In non-tracing builds, if every sub-block supports shared encoding,- // we can run the entire layer in a single compute encoder.- #[cfg(not(feature = "tracing"))]- {- if self.supports_shared_encoder() {- command_buffer- .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));- return Ok(());- }- }-#[cfg(feature = "tracing")]let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();@@ -384,103 +371,4 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {let _ = parameters;Ok(())}-- fn supports_shared_encoder(&self) -> bool {- #[cfg(feature = "tracing")]- {- false- }-- #[cfg(not(feature = "tracing"))]- {- let mixer_ok = match &self.mixer {- MixerExecutables::Attention {- qkv_projection,- qk_norm,- rope,- attention,- out_projection,- } => {- qkv_projection.supports_shared_encoder()- && qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)- && rope.supports_shared_encoder()- && attention.supports_shared_encoder()- && out_projection.supports_shared_encoder()- },- MixerExecutables::StateSpace {- mixer,- } => mixer.supports_shared_encoder(),- MixerExecutables::ShortConv {- mixer,- } => mixer.supports_shared_encoder(),- };-- self.copy_main_to_shortcut.supports_shared_encoder()- && self.pre_attention_norm.supports_shared_encoder()- && mixer_ok- && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)- && self.main_shortcut_add_swap.supports_shared_encoder()- && self.pre_mlp_norm.supports_shared_encoder()- && self.mlp.supports_shared_encoder()- && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)- }- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {- debug_assert!(- self.supports_shared_encoder(),- "encode_with_shared_encoder called on unsupported LayerExecutables"- );-- self.copy_main_to_shortcut.encode_with_shared_encoder(state, parameters, encoder);- self.pre_attention_norm.encode_with_shared_encoder(state, parameters, encoder);-- match &self.mixer {- MixerExecutables::Attention {- qkv_projection,- qk_norm,- rope,- attention,- out_projection,- } => {- qkv_projection.encode_with_shared_encoder(state, parameters, encoder);- if let Some(norm) = qk_norm {- norm.encode_with_shared_encoder(state, parameters, encoder);- }- rope.encode_with_shared_encoder(state, parameters, encoder);- attention.encode_with_shared_encoder(state, parameters, encoder);- out_projection.encode_with_shared_encoder(state, parameters, encoder);- },- MixerExecutables::StateSpace {- mixer,- } => {- mixer.encode_with_shared_encoder(state, parameters, encoder);- },- MixerExecutables::ShortConv {- mixer,- } => {- mixer.encode_with_shared_encoder(state, parameters, encoder);- },- }-- if let Some(post_attention_norm) = &self.post_attention_norm {- post_attention_norm.encode_with_shared_encoder(state, parameters, encoder);- }-- self.main_shortcut_add_swap.encode_with_shared_encoder(state, parameters, encoder);-- self.pre_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);- self.mlp.encode_with_shared_encoder(state, parameters, encoder);-- if let Some(post_mlp_norm) = &self.post_mlp_norm {- post_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);- }-- self.main_shortcut_add_swap.encode_with_shared_encoder(state, parameters, encoder);- }}diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rsindex a622ffe73..0c2ec7e55 100644--- a/crates/uzu/src/encodable_block/layer_norm.rs+++ b/crates/uzu/src/encodable_block/layer_norm.rs@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{Kernels, LayerNormKernel},},config::{NormalizationConfig, UpcastMode},@@ -33,7 +33,7 @@ pub struct LayerNorm<B: Backend> {config: NormalizationConfig,input_array_id: ArrayId,output_array_id: ArrayId,- scales_buffer: Rc<RefCell<B::NativeBuffer>>,+ scales_buffer: Rc<RefCell<B::Buffer>>,}impl<B: Backend> LayerNorm<B> {@@ -71,16 +71,12 @@ impl<B: Backend> LayerNorm<B> {}impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);@@ -111,7 +107,8 @@ impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),full_layer,- encoder,+ command_buffer,);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex 318fd7f28..75948b15e 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -139,34 +139,10 @@ impl<B: Backend> MambaMixer<B> {}impl<B: Backend> MambaMixer<B> {- fn encode_pipeline_with_encoder(- &self,- state: &mut ForwardPassState<B>,- encoder: &mut B::ComputeEncoder,- parameters: &EncodingParameters<B>,- ) {- let active_suffix_length = state.active_suffix_length();- if active_suffix_length == 0 {- return;- }-- self.in_projection.encode_with_shared_encoder(state, parameters, encoder);- self.run_split_inproj(state, encoder, active_suffix_length);- self.run_conv_scan(state, encoder, active_suffix_length);-- if active_suffix_length == 1 {- self.run_decode_ssm(state, encoder, active_suffix_length);- } else {- self.run_prefill_ssm(state, encoder, active_suffix_length);- }-- self.out_projection.encode_with_shared_encoder(state, parameters, encoder);- }-fn run_split_inproj(&self,state: &mut ForwardPassState<B>,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,suffix_length: usize,) {let arrays = state.arrays(&[@@ -199,14 +175,14 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,inner_dim as u32,num_heads as u32,- encoder,+ command_buffer,)}fn run_conv_scan(&self,state: &mut ForwardPassState<B>,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,suffix_length: usize,) {let arrays = state.arrays(&[@@ -238,7 +214,7 @@ impl<B: Backend> MambaMixer<B> {conv_inputs.buffer().borrow().deref(),weight_buf_borrow.deref(),bias_buf_borrow.as_deref(),- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,x_arr.buffer().borrow_mut().deref_mut(),b_arr.buffer().borrow_mut().deref_mut(),c_arr.buffer().borrow_mut().deref_mut(),@@ -250,7 +226,7 @@ impl<B: Backend> MambaMixer<B> {suffix_length as u32,inner_dim as u32,proj_dim as u32,- encoder,+ command_buffer,);}} else {@@ -266,7 +242,7 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,suffix_length as u32,conv_dim as u32,- encoder,+ command_buffer,);Some(buffer)@@ -278,7 +254,7 @@ impl<B: Backend> MambaMixer<B> {let padded_borrow = padded_buf.as_ref().map(|b| b.borrow());let conv_inputs_borrow = conv_inputs.buffer();let conv_inputs_ref = conv_inputs_borrow.borrow();- let conv_source: &B::NativeBuffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());+ let conv_source: &B::Buffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());self.conv_scan.encode(conv_source,weight_buf_borrow.deref(),@@ -294,7 +270,7 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,inner_dim as u32,proj_dim as u32,- encoder,+ command_buffer,)}}@@ -303,7 +279,7 @@ impl<B: Backend> MambaMixer<B> {fn run_prefill_ssm(&self,state: &mut ForwardPassState<B>,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,suffix_length: usize,) {let base_arrays = state.arrays(&[@@ -327,7 +303,7 @@ impl<B: Backend> MambaMixer<B> {let skip_rc = self.skip_connection_weight.buffer();let skip_borrow = skip_rc.borrow();self.ssd_prefill.encode(- encoder,+ command_buffer,SSDPrefillArguments {x: x.buffer().borrow().deref(),dt: dt.buffer().borrow().deref(),@@ -354,7 +330,7 @@ impl<B: Backend> MambaMixer<B> {fn run_decode_ssm(&self,state: &mut ForwardPassState<B>,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,suffix_length: usize,) {let arrays = state.arrays(&[@@ -396,7 +372,7 @@ impl<B: Backend> MambaMixer<B> {c.buffer().borrow().deref(),skip_borrow.deref(),z.buffer().borrow().deref(),- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,y.buffer().borrow_mut().deref_mut(),state_arr.buffer().borrow_mut().deref_mut(),group_size as u32,@@ -408,7 +384,7 @@ impl<B: Backend> MambaMixer<B> {suffix_length as u32,h,dh,- encoder,+ command_buffer,);}@@ -438,56 +414,23 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- if self.supports_shared_encoder() {- command_buffer- .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));-- if parameters.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;- }- return Ok(());- }-let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {return Ok(());}self.in_projection.encode(state, parameters, command_buffer)?;-- command_buffer.with_compute_encoder(|encoder| {- self.run_split_inproj(state, encoder, active_suffix_length);- self.run_conv_scan(state, encoder, active_suffix_length);- if active_suffix_length == 1 {- self.run_decode_ssm(state, encoder, active_suffix_length);- } else {- self.run_prefill_ssm(state, encoder, active_suffix_length);- }- });-- self.out_projection.encode(state, parameters, command_buffer)?;-- if parameters.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;+ self.run_split_inproj(state, command_buffer, active_suffix_length);+ self.run_conv_scan(state, command_buffer, active_suffix_length);+ if active_suffix_length == 1 {+ self.run_decode_ssm(state, command_buffer, active_suffix_length);+ } else {+ self.run_prefill_ssm(state, command_buffer, active_suffix_length);}+ self.out_projection.encode(state, parameters, command_buffer)?;Ok(())}-- fn supports_shared_encoder(&self) -> bool {- self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {- self.encode_pipeline_with_encoder(state, encoder, parameters);- }}diff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rsindex df0d31348..36c41f65b 100644--- a/crates/uzu/src/encodable_block/mlp.rs+++ b/crates/uzu/src/encodable_block/mlp.rs@@ -32,54 +32,11 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {fn encode(&self,state: &mut ForwardPassState<B>,- params: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ params: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- if self.supports_shared_encoder() {- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, params, encoder));- } else {- // Up- self.up.encode(state, params, command_buffer)?;-- // Gate act+mul (fused_up -> hidden)- command_buffer.with_compute_encoder(|encoder| {- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);- let fused = arrays[0].borrow_mut();- let hidden = arrays[1].borrow_mut();- let m = fused.shape()[0] as i32;- let fused_buf_rc = fused.buffer();- let fused_buf_borrow = fused_buf_rc.borrow();- let hidden_buf_rc = hidden.buffer();- let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();-- self.gate- .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)- .expect("Failed to encode MLP activation/mul kernel");- });-- // Down- self.down.encode(state, params, command_buffer)?;- }-- if params.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;- }- Ok(())- }-- fn supports_shared_encoder(&self) -> bool {- self.up.supports_shared_encoder() && self.down.supports_shared_encoder()- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- params: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {// Up- self.up.encode_with_shared_encoder(state, params, encoder);+ self.up.encode(state, params, command_buffer)?;// Gate act+mul (fused_up -> hidden)let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);@@ -91,7 +48,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {let hidden_buf_rc = hidden.buffer();let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();self.gate- .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)+ .encode(command_buffer, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m).expect("Failed to encode MLP activation/mul kernel");drop(hidden_buf_borrow);drop(fused_buf_borrow);@@ -99,6 +56,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {drop(hidden);// Down- self.down.encode_with_shared_encoder(state, params, encoder);+ self.down.encode(state, params, command_buffer)?;+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 27847edb9..a973d83c3 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -58,26 +58,14 @@ pub use tensor_copy::TensorCopy;pub use transformer_layer::{embed_block, linear_block};#[derive(Clone)]-pub struct EncodingParameters<'a, B: Backend> {- pub warmup: bool,- pub enable_commit: bool,- pub wait_until_completed: bool,+pub struct EncodingParameters {pub projection_step: Option<usize>,- pub predicate: Option<&'a B::NativeBuffer>,}-impl<'a, B: Backend> EncodingParameters<'a, B> {- pub fn new(- warmup: bool,- enable_commit: bool,- wait_until_completed: bool,- ) -> Self {+impl EncodingParameters {+ pub fn new() -> Self {Self {- warmup,- enable_commit,- wait_until_completed,projection_step: None,- predicate: None,}}@@ -88,39 +76,13 @@ impl<'a, B: Backend> EncodingParameters<'a, B> {self.projection_step = Some(projection_step);self}-- pub fn with_predicate(- mut self,- predicate: &'a B::NativeBuffer,- ) -> Self {- self.predicate = Some(predicate);- self- }-- pub fn predicate_ref(&self) -> Option<&B::NativeBuffer> {- self.predicate- }}pub trait EncodableBlock<B: Backend> {- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- );-- fn supports_shared_encoder(&self) -> bool {- false- }-fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,- ) -> Result<(), B::Error> {- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));- Ok(())- }+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error>;}diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rsindex 07869e271..a5be081ba 100644--- a/crates/uzu/src/encodable_block/moe_block.rs+++ b/crates/uzu/src/encodable_block/moe_block.rs@@ -12,7 +12,7 @@ use crate::{Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,array::Array,backends::common::{- Backend, CommandBuffer, CopyEncoder, Kernels,+ Backend, CommandBuffer, CommandBufferEncoding, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,@@ -28,16 +28,16 @@ use crate::{};struct RouterBlock<B: Backend> {- weights_buf: Rc<RefCell<B::NativeBuffer>>,- biases_buf: Rc<RefCell<B::NativeBuffer>>,+ weights_buf: Rc<RefCell<B::Buffer>>,+ biases_buf: Rc<RefCell<B::Buffer>>,}#[derive(Clone)]struct SharedMoeWeights<B: Backend> {- pub w13_buf: Rc<RefCell<B::NativeBuffer>>,- pub w2_buf: Rc<RefCell<B::NativeBuffer>>,- pub up_biases_buf: Rc<RefCell<B::NativeBuffer>>,- pub down_biases_buf: Rc<RefCell<B::NativeBuffer>>,+ pub w13_buf: Rc<RefCell<B::Buffer>>,+ pub w2_buf: Rc<RefCell<B::Buffer>>,+ pub up_biases_buf: Rc<RefCell<B::Buffer>>,+ pub down_biases_buf: Rc<RefCell<B::Buffer>>,}pub struct MoeBlock<B: Backend> {@@ -192,8 +192,8 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let suffix_length = state.active_suffix_length();let arrays = state.arrays(&[@@ -219,7 +219,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {ArrayId::MoeTwoPassRowExpertMap,]);- let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<RefCell<B::NativeBuffer>> { array.borrow().buffer() };+ let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<RefCell<B::Buffer>> { array.borrow().buffer() };let mut array_iter = arrays.iter();let main_buf = clone_buffer(array_iter.next().unwrap());@@ -249,39 +249,37 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {let k_tile = 128;// Clear internal MoE buffers- command_buffer.with_copy_encoder(|encoder| {- if suffix_length > 0 && k > 0 {- let entries = suffix_length * k;- let topk_bytes = entries * std::mem::size_of::<u32>();- let tok2row_bytes = entries * std::mem::size_of::<i32>();-- // Clear topk_ids and tok2row buffers- if topk_bytes > 0 {- encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);- }- if tok2row_bytes > 0 {- encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);- }+ if suffix_length > 0 && k > 0 {+ let entries = suffix_length * k;+ let topk_bytes = entries * std::mem::size_of::<u32>();+ let tok2row_bytes = entries * std::mem::size_of::<i32>();++ // Clear topk_ids and tok2row buffers+ if topk_bytes > 0 {+ command_buffer.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);+ }+ if tok2row_bytes > 0 {+ command_buffer.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);+ }- // Clear hidden buffer- let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();- if hidden_bytes > 0 {- encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);- }+ // Clear hidden buffer+ let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();+ if hidden_bytes > 0 {+ command_buffer.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);+ }- // Clear y_partial buffer- let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();- if y_partial_bytes > 0 {- encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);- }+ // Clear y_partial buffer+ let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();+ if y_partial_bytes > 0 {+ command_buffer.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);+ }- // Clear x_perm buffer- let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();- if x_perm_bytes > 0 {- encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);- }+ // Clear x_perm buffer+ let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();+ if x_perm_bytes > 0 {+ command_buffer.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);}- });+ }// Use fused Router+TopK kernel for non-quantized routersif self.model_dim % 4 != 0 {@@ -295,69 +293,61 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {}if suffix_length > 0 && e > 0 && k > 0 {- command_buffer.with_compute_encoder(|encoder| {- // Use the fused router+topk kernel- self.router_topk_kernel.encode(- main_buf.borrow().deref(),- self.router.weights_buf.borrow().deref(),- self.router.biases_buf.borrow().deref(),- topk_ids_buf.borrow_mut().deref_mut(),- topk_probs_buf.borrow_mut().deref_mut(),- suffix_length as u32,- self.model_dim as u32,- e as u32,- k as u32,- self.router_renorm,- encoder,- );- });- }-- command_buffer.with_compute_encoder(|encoder| {- self.counts_offsets_kernel.encode(- topk_ids_buf.borrow().deref(),- offsets_buf.borrow_mut().deref_mut(),- sumk_buf.borrow_mut().deref_mut(),- partials_buf.borrow_mut().deref_mut(),+ // Use the fused router+topk kernel+ self.router_topk_kernel.encode(+ main_buf.borrow().deref(),+ self.router.weights_buf.borrow().deref(),+ self.router.biases_buf.borrow().deref(),+ topk_ids_buf.borrow_mut().deref_mut(),+ topk_probs_buf.borrow_mut().deref_mut(),suffix_length as u32,+ self.model_dim as u32,e as u32,k as u32,- encoder,+ self.router_renorm,+ command_buffer,);- });+ }++ self.counts_offsets_kernel.encode(+ topk_ids_buf.borrow().deref(),+ offsets_buf.borrow_mut().deref_mut(),+ sumk_buf.borrow_mut().deref_mut(),+ partials_buf.borrow_mut().deref_mut(),+ suffix_length as u32,+ e as u32,+ k as u32,+ command_buffer,+ );let num_blocks = ((suffix_length + 255) / 256).max(1);let num_tiles = ((e + 512 - 1) / 512).max(1);- command_buffer.with_compute_encoder(|encoder| {- self.scatter_bases_kernel.encode(- partials_buf.borrow().deref(),- block_bases_buf.borrow_mut().deref_mut(),- block_alloc_buf.borrow_mut().deref_mut(),- e as u32,- num_blocks as u32,- num_tiles as u32,- 0u32,- encoder,- );- });- command_buffer.with_compute_encoder(|encoder| {- self.scatter_map_kernel.encode(- topk_ids_buf.borrow().deref(),- topk_probs_buf.borrow().deref(),- offsets_buf.borrow().deref(),- block_bases_buf.borrow().deref(),- block_alloc_buf.borrow().deref(),- bucketed_ids_buf.borrow_mut().deref_mut(),- bucketed_probs_buf.borrow_mut().deref_mut(),- suffix_length as u32,- e as u32,- k as u32,- num_blocks as u32,- num_tiles as u32,- tok2row_buf.borrow_mut().deref_mut(),- encoder,- );- });+ self.scatter_bases_kernel.encode(+ partials_buf.borrow().deref(),+ block_bases_buf.borrow_mut().deref_mut(),+ block_alloc_buf.borrow_mut().deref_mut(),+ e as u32,+ num_blocks as u32,+ num_tiles as u32,+ 0u32,+ command_buffer,+ );+ self.scatter_map_kernel.encode(+ topk_ids_buf.borrow().deref(),+ topk_probs_buf.borrow().deref(),+ offsets_buf.borrow().deref(),+ block_bases_buf.borrow().deref(),+ block_alloc_buf.borrow().deref(),+ bucketed_ids_buf.borrow_mut().deref_mut(),+ bucketed_probs_buf.borrow_mut().deref_mut(),+ suffix_length as u32,+ e as u32,+ k as u32,+ num_blocks as u32,+ num_tiles as u32,+ tok2row_buf.borrow_mut().deref_mut(),+ command_buffer,+ );self.gather_kernels.encode(command_buffer,@@ -467,36 +457,17 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {self.experts_two_pass_prefill_kernel.encode(command_buffer, args);}- command_buffer.with_compute_encoder(|encoder| {- self.finalize_kernel.encode(- tok2row_buf.borrow().deref(),- topk_probs_buf.borrow().deref(),- y_partial_buf.borrow().deref(),- main_buf.borrow_mut().deref_mut(),- suffix_length as u32,- self.model_dim as u32,- k as u32,- encoder,- );- });+ self.finalize_kernel.encode(+ tok2row_buf.borrow().deref(),+ topk_probs_buf.borrow().deref(),+ y_partial_buf.borrow().deref(),+ main_buf.borrow_mut().deref_mut(),+ suffix_length as u32,+ self.model_dim as u32,+ k as u32,+ command_buffer,+ );- if parameters.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;- }Ok(())}-- fn supports_shared_encoder(&self) -> bool {- false- }-- fn encode_with_shared_encoder(- &self,- _state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- _encoder: &mut B::ComputeEncoder,- ) {- unreachable!("MoeBlock does not support shared compute encoder");- }}diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rsindex 0553ecef4..308b3fab6 100644--- a/crates/uzu/src/encodable_block/normalization.rs+++ b/crates/uzu/src/encodable_block/normalization.rs@@ -5,7 +5,7 @@ use thiserror::Error;use super::{EncodableBlock, EncodingParameters, LayerNorm, LayerNormError, RMSNorm, RMSNormError};use crate::{DataType,- backends::common::Backend,+ backends::common::{Backend, CommandBuffer},config::NormalizationConfig,forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -59,22 +59,15 @@ impl<B: Backend> Normalization<B> {}impl<B: Backend> EncodableBlock<B> for Normalization<B> {- fn supports_shared_encoder(&self) -> bool {- match self {- Self::LayerNorm(layer_norm) => layer_norm.supports_shared_encoder(),- Self::RMSNorm(rms_norm) => rms_norm.supports_shared_encoder(),- }- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {match self {- Self::LayerNorm(layer_norm) => layer_norm.encode_with_shared_encoder(state, parameters, encoder),- Self::RMSNorm(rms_norm) => rms_norm.encode_with_shared_encoder(state, parameters, encoder),+ Self::LayerNorm(layer_norm) => layer_norm.encode(state, parameters, command_buffer),+ Self::RMSNorm(rms_norm) => rms_norm.encode(state, parameters, command_buffer),}}}diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rsindex 00caf4324..bf8010fac 100644--- a/crates/uzu/src/encodable_block/pooling.rs+++ b/crates/uzu/src/encodable_block/pooling.rs@@ -21,16 +21,16 @@ enum PoolingKernel<B: Backend> {impl<B: Backend> PoolingKernel<B> {fn encode(&self,- input: &B::NativeBuffer,- output: &mut B::NativeBuffer,+ input: &B::Buffer,+ output: &mut B::Buffer,seq_len: u32,hidden_dim: u32,batch_size: u32,- encoder: &mut B::ComputeEncoder,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) {match self {- Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),- Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),+ Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),+ Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),}}}@@ -64,29 +64,9 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));-- #[cfg(feature = "tracing")]- {- let output_pooling_trace = state.traces().borrow().output_pooling().clone();- state.encode_copy_array(command_buffer, ArrayId::ClassifierPooling, output_pooling_trace);- }- Ok(())- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {let batch_size = 1;let seq_len = state.aux_buffers_suffix_length();@@ -100,7 +80,14 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {seq_len as u32,self.model_dim as u32,batch_size,- encoder,+ command_buffer,);++ #[cfg(feature = "tracing")]+ {+ let output_pooling_trace = state.traces().borrow().output_pooling().clone();+ state.encode_copy_array(command_buffer, ArrayId::ClassifierPooling, output_pooling_trace);+ }+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rsindex c41133e6c..46ec73491 100644--- a/crates/uzu/src/encodable_block/prediction_head.rs+++ b/crates/uzu/src/encodable_block/prediction_head.rs@@ -39,17 +39,13 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- if self.supports_shared_encoder() {- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));- } else {- self.dense.encode(state, parameters, command_buffer)?;- self.activation.encode(state, parameters, command_buffer)?;- self.norm.encode(state, parameters, command_buffer)?;- self.readout.encode(state, parameters, command_buffer)?;- }+ self.dense.encode(state, parameters, command_buffer)?;+ self.activation.encode(state, parameters, command_buffer)?;+ self.norm.encode(state, parameters, command_buffer)?;+ self.readout.encode(state, parameters, command_buffer)?;#[cfg(feature = "tracing")]{@@ -61,29 +57,6 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {);}- if parameters.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;- }Ok(())}-- fn supports_shared_encoder(&self) -> bool {- self.dense.supports_shared_encoder()- && self.activation.supports_shared_encoder()- && self.norm.supports_shared_encoder()- && self.readout.supports_shared_encoder()- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {- self.dense.encode_with_shared_encoder(state, parameters, encoder);- self.activation.encode_with_shared_encoder(state, parameters, encoder);- self.norm.encode_with_shared_encoder(state, parameters, encoder);- self.readout.encode_with_shared_encoder(state, parameters, encoder);- }}diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rsindex 4c8bb857a..0c25f43bf 100644--- a/crates/uzu/src/encodable_block/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{Kernels, QKNormKernel},},config::{NormalizationConfig, UpcastMode},@@ -34,8 +34,8 @@ pub struct QKNorm<B: Backend> {query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,qkv_array_id: ArrayId,- query_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,- key_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,+ query_scales_buffer: Option<Rc<RefCell<B::Buffer>>>,+ key_scales_buffer: Option<Rc<RefCell<B::Buffer>>>,num_q_heads: usize,num_kv_heads: usize,head_dim: usize,@@ -124,16 +124,12 @@ impl<B: Backend> QKNorm<B> {}impl<B: Backend> EncodableBlock<B> for QKNorm<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let qkv_binding = state.arrays(&[self.qkv_array_id]);let qkv_shape = {let qkv_array = qkv_binding[0].borrow();@@ -148,7 +144,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {(&self.query_kernel, &self.query_scales_buffer, &self.query_config){query_kernel.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,query_scales_buffer.borrow().deref(),qkv_array.buffer().borrow_mut().deref_mut(),batch_size,@@ -160,7 +156,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {0,self.num_q_heads as u32,query_config.upcast_mode == UpcastMode::FullLayer,- encoder,+ command_buffer,);}@@ -169,7 +165,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {(&self.key_kernel, &self.key_scales_buffer, &self.key_config){key_kernel.encode(- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,key_scales_buffer.borrow().deref(),qkv_array.buffer().borrow_mut().deref_mut(),batch_size,@@ -181,8 +177,9 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {self.num_q_heads as u32,self.num_kv_heads as u32,key_config.upcast_mode == UpcastMode::FullLayer,- encoder,+ command_buffer,);}+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rsindex ff9494f18..ac50416da 100644--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs+++ b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend, Context, NativeBuffer,+ Backend, Buffer, CommandBuffer, Context,kernel::{Kernels, QuantizedEmbeddingLookupKernel},},config::QuantizationMode,@@ -71,9 +71,9 @@ pub enum QuantizedEmbeddingLookupError<B: Backend> {pub struct QuantizedEmbeddingLookup<B: Backend> {kernel: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,- weights_buffer: Rc<RefCell<B::NativeBuffer>>,- scales_buffer: Rc<RefCell<B::NativeBuffer>>,- biases_buffer: Rc<RefCell<B::NativeBuffer>>,+ weights_buffer: Rc<RefCell<B::Buffer>>,+ scales_buffer: Rc<RefCell<B::Buffer>>,+ biases_buffer: Rc<RefCell<B::Buffer>>,mode: QuantizationMode,input_scale: f32,vocab_size: u32,@@ -239,16 +239,12 @@ impl<B: Backend> QuantizedEmbeddingLookup<B> {}impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);let batch_size = state.active_suffix_length();let token_ids_array = arrays[0].borrow_mut();@@ -272,7 +268,8 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {self.group_size,self.input_scale,quant_mode,- encoder,+ command_buffer,);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rsindex 7abaa606f..791237300 100644--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend, Context, NativeBuffer,+ Backend, Buffer, CommandBuffer, Context,kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,QuantizedMatmulKernelEncodable, QuantizedMatmulType,@@ -71,9 +71,9 @@ pub enum QuantizedEmbeddingReadoutError<B: Backend> {pub struct QuantizedEmbeddingReadout<B: Backend> {kernel: QuantizedMatmulKernelEncodable<B>,- weights_buffer: Rc<RefCell<B::NativeBuffer>>,- scales_buffer: Rc<RefCell<B::NativeBuffer>>,- biases_buffer: Rc<RefCell<B::NativeBuffer>>,+ weights_buffer: Rc<RefCell<B::Buffer>>,+ scales_buffer: Rc<RefCell<B::Buffer>>,+ biases_buffer: Rc<RefCell<B::Buffer>>,vocab_size: usize,model_dim: usize,}@@ -229,20 +229,16 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {}impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);let batch_size = state.sampling_length();if batch_size == 0 {- return;+ return Ok(());}let sampling_start = state.sampling_start();@@ -254,7 +250,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {self.kernel.encode(- encoder,+ command_buffer,QuantizedMatmulArguments {a_buffer: input_array.buffer().borrow().deref(),a_offset,@@ -269,5 +265,6 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {},).expect("Failed to encode quantized embedding readout kernel");+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rsindex 173d53320..c2fce2e9c 100644--- a/crates/uzu/src/encodable_block/quantized_linear.rs+++ b/crates/uzu/src/encodable_block/quantized_linear.rs@@ -10,7 +10,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{Kernels, TensorAddBiasKernel,quant_matmul::{@@ -88,10 +88,10 @@ pub enum QuantizedLinearError<B: Backend> {pub struct QuantizedLinear<B: Backend> {kernel: QuantizedMatmulKernelEncodable<B>,bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,- biases_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,- weights_buffer: Rc<RefCell<B::NativeBuffer>>,- scales_buffer: Rc<RefCell<B::NativeBuffer>>,- zero_points_or_biases_buffer: Rc<RefCell<B::NativeBuffer>>,+ biases_buffer: Option<Rc<RefCell<B::Buffer>>>,+ weights_buffer: Rc<RefCell<B::Buffer>>,+ scales_buffer: Rc<RefCell<B::Buffer>>,+ zero_points_or_biases_buffer: Rc<RefCell<B::Buffer>>,quantization_type: QuantizedMatmulType,input_dim: usize,output_dim: usize,@@ -245,16 +245,12 @@ impl<B: Backend> QuantizedLinear<B> {}impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();let input_array = arrays[0].borrow_mut();@@ -264,7 +260,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {self.kernel.encode(- encoder,+ command_buffer,QuantizedMatmulArguments {a_buffer: input_array.buffer().borrow().deref(),a_offset: 0,@@ -282,15 +278,15 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {let total_length = batch_size * self.output_dim;- bias_add_kernel.encode_if(- None::<&B::NativeBuffer>,+ bias_add_kernel.encode(+ None::<&B::Buffer>,biases_buffer.borrow().deref(),output_buf_borrow.deref_mut(),self.output_dim as u32,total_length as u32,- encoder,- parameters.predicate_ref(),+ command_buffer,);}+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 699787612..13a1c7483 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{Kernels, RMSNormKernel},},config::{NormalizationConfig, UpcastMode},@@ -33,7 +33,7 @@ pub struct RMSNorm<B: Backend> {config: NormalizationConfig,input_array_id: ArrayId,output_array_id: ArrayId,- scales_buffer: Rc<RefCell<B::NativeBuffer>>,+ scales_buffer: Rc<RefCell<B::Buffer>>,use_sampling_range: bool,}@@ -85,16 +85,12 @@ impl<B: Backend> RMSNorm<B> {}impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);@@ -117,7 +113,7 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {};let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));if batch_len == 0 {- return;+ return Ok(());}let row_size_in_bytes = input_shape[1] * input_elem_size;@@ -138,7 +134,8 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),self.config.upcast_mode == UpcastMode::FullLayer,- encoder,+ command_buffer,);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rsindex 6660fbe67..66ca919f2 100644--- a/crates/uzu/src/encodable_block/rope.rs+++ b/crates/uzu/src/encodable_block/rope.rs@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{Kernels, RopeKernel},},forward_pass::state::{ArrayId, ForwardPassState, RopeType},@@ -31,16 +31,12 @@ impl<B: Backend> Rope<B> {}impl<B: Backend> EncodableBlock<B> for Rope<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);let qkv_array = qkv_binding[0].borrow();@@ -95,7 +91,8 @@ impl<B: Backend> EncodableBlock<B> for Rope<B> {num_groups as u32,suffix_length as u32,rope_max_seq_len as u32,- encoder,- )+ command_buffer,+ );+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex 779c3fcd2..2b4901f0b 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},},forward_pass::state::{ArrayId, ForwardPassState},@@ -45,12 +45,12 @@ impl<B: Backend> Sampling<B> {}impl<B: Backend> EncodableBlock<B> for Sampling<B> {- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");let logits_binding = state.arrays(&[ArrayId::Logits]);@@ -59,7 +59,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {let logits_shape = logits.shape();let batch_size = state.sampling_length();if batch_size == 0 {- return;+ return Ok(());}let sampling_start = state.sampling_start();let vocab_size = logits_shape[1];@@ -85,7 +85,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {let seeds_buf_borrow = seeds_buf_rc.borrow();let output_buf_rc = output_buffer_ref.buffer();let mut output_buf_borrow = output_buf_rc.borrow_mut();- if let Err(e) = self.kernel.encode_with_encoder(+ if let Err(e) = self.kernel.encode(logits_buf_borrow.deref_mut(),Some(seeds_buf_borrow.deref()),seeds_offset,@@ -95,9 +95,10 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {sampling_method,batch_size,vocab_size,- encoder,+ command_buffer,) {panic!("Sampling encoding failed: {:?}", e);}+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex c6e272ccc..75534176f 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -103,48 +103,6 @@ impl<B: Backend> ShortConvMixer<B> {}impl<B: Backend> ShortConvMixer<B> {- fn encode_pipeline(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,- ) -> Result<(), B::Error> {- let active_suffix_length = state.active_suffix_length();- if active_suffix_length == 0 {- return Ok(());- }-- self.in_projection.encode(state, parameters, command_buffer)?;-- command_buffer.with_compute_encoder(|encoder| self.run_conv(state, encoder, active_suffix_length));-- self.out_projection.encode(state, parameters, command_buffer)?;-- if parameters.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;- }- Ok(())- }-- fn encode_pipeline_with_encoder(- &self,- state: &mut ForwardPassState<B>,- encoder: &mut B::ComputeEncoder,- parameters: &EncodingParameters<B>,- ) {- let active_suffix_length = state.active_suffix_length();- if active_suffix_length == 0 {- return;- }-- self.in_projection.encode_with_shared_encoder(state, parameters, encoder);-- self.run_conv(state, encoder, active_suffix_length);-- self.out_projection.encode_with_shared_encoder(state, parameters, encoder);- }-fn clear_suffix_state_valid_range(&self,state: &ForwardPassState<B>,@@ -174,7 +132,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut B::ComputeEncoder,+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,active_suffix_length: usize,) {self.clear_suffix_state_valid_range(state);@@ -213,7 +171,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_prefill_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut B::ComputeEncoder,+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -270,7 +228,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_trie_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut B::ComputeEncoder,+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,sampling_start: usize,trie_len: usize,) {@@ -329,7 +287,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_decode_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut B::ComputeEncoder,+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -353,7 +311,7 @@ impl<B: Backend> ShortConvMixer<B> {in_proj.buffer().borrow().deref(),decode_weight_buf_borrow.deref(),decode_bias_buf_borrow.as_deref(),- None::<&B::NativeBuffer>,+ None::<&B::Buffer>,out.buffer().borrow_mut().deref_mut(),conv_state.buffer().borrow_mut().deref_mut(),suffix_length as u32,@@ -370,33 +328,20 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- command_buffer: &mut B::CommandBuffer,+ parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- if self.supports_shared_encoder() {- command_buffer- .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));-- if parameters.wait_until_completed {- command_buffer.submit();- command_buffer.wait_until_completed()?;- }- } else {- self.encode_pipeline(state, parameters, command_buffer)?;+ let active_suffix_length = state.active_suffix_length();+ if active_suffix_length == 0 {+ return Ok(());}- Ok(())- }- fn supports_shared_encoder(&self) -> bool {- self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()- }+ self.in_projection.encode(state, parameters, command_buffer)?;- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {- self.encode_pipeline_with_encoder(state, encoder, parameters);+ self.run_conv(state, command_buffer, active_suffix_length);++ self.out_projection.encode(state, parameters, command_buffer)?;++ Ok(())}}diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rsindex 961df3955..627208b12 100644--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{Kernels, TensorAddSwapKernel},},forward_pass::state::{ArrayId, ForwardPassState},@@ -32,16 +32,12 @@ impl<B: Backend> TensorAddSwap<B> {}impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");@@ -54,7 +50,8 @@ impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {skip_array.buffer().borrow_mut().deref_mut(),main_array.buffer().borrow_mut().deref_mut(),length as u32,- encoder,+ command_buffer,);+ Ok(())}}diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rsindex c8ab574ff..d82361ead 100644--- a/crates/uzu/src/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/encodable_block/tensor_copy.rs@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{- Backend,+ Backend, CommandBuffer,kernel::{Kernels, TensorCopyKernel},},forward_pass::state::{ArrayId, ForwardPassState},@@ -32,16 +32,12 @@ impl<B: Backend> TensorCopy<B> {}impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(+ fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters<B>,- encoder: &mut B::ComputeEncoder,- ) {+ _parameters: &EncodingParameters,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error> {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");@@ -54,7 +50,8 @@ impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {source_array.buffer().borrow().deref(),destination_array.buffer().borrow_mut().deref_mut(),length as u32,- encoder,+ command_buffer,);+ Ok(())}}diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rsindex e3485cd81..8a88a6741 100644--- a/crates/uzu/src/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -5,7 +5,7 @@ use std::{use crate::{array::{Array, ArrayContextExt},- backends::common::{Backend, kernel::kv_cache_update::KVCacheUpdate},+ backends::common::{Backend, CommandBuffer, kernel::kv_cache_update::KVCacheUpdate},config::DecoderLayerType,forward_pass::{kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},@@ -276,7 +276,7 @@ impl<B: Backend> CacheLayers<B> {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,kv_cache_update: &KVCacheUpdate<B>,) {let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rsindex def9925c7..254bc84d8 100644--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs@@ -1,7 +1,7 @@use crate::{array::{Array, ArrayCell, ArrayContextExt},backends::common::{- Backend,+ Backend, CommandBuffer,kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},},utils::attention::fill_attention_bias,@@ -163,7 +163,7 @@ impl<B: Backend> KVCacheLayer<B> {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,kv_cache_update: &KVCacheUpdate<B>,) {match &mut self.state {@@ -213,7 +213,7 @@ impl<B: Backend> KVCacheLayer<B> {&self,source_indices: &[usize],destination_indices: &[usize],- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,kv_cache_update: &KVCacheUpdate<B>,) {if source_indices == destination_indices {diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex b78a8c777..c04a18ad9 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -10,7 +10,7 @@ use crate::forward_pass::traces::ActivationTrace;use crate::{DataType, DecoderConfig,array::{Array, ArrayCell, ArrayCellExt},- backends::common::{Backend, CommandBuffer, Context, CopyEncoder},+ backends::common::{Backend, CommandBuffer, CommandBufferEncoding, Context},forward_pass::{cache_layers::CacheLayers,model_shape::ModelShape,@@ -157,8 +157,8 @@ impl<B: Backend> ForwardPassState<B> {external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,skip_token_ids_copy: bool,should_fill_attention_bias: bool,- async_positions: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,- async_seeds: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,+ async_positions: Option<(Rc<RefCell<B::Buffer>>, usize)>,+ async_seeds: Option<(Rc<RefCell<B::Buffer>>, usize)>,) -> Self {let suffix_length = token_ids.len();assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");@@ -777,7 +777,7 @@ impl<B: Backend> ForwardPassState<B> {pub fn encode_copy_array(&self,- command_buffer: &mut B::CommandBuffer,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,source_array_id: ArrayId,destination_array: RefCell<Array<B>>,) {@@ -791,8 +791,6 @@ impl<B: Backend> ForwardPassState<B> {let copy_size_bytes = dst_borrow.size();debug_assert_eq!(dst_borrow.size(), src_borrow.size());- command_buffer.with_copy_encoder(|encoder| {- encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);- });+ command_buffer.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);}}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 7d59e7404..8a3685efd 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -19,7 +19,8 @@ use super::{};use crate::{backends::common::{- Backend, CommandBuffer, Context, NativeBuffer,+ Backend, Buffer, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending, Context,kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},},encodable_block::{EncodableBlock, EncodingParameters},@@ -487,8 +488,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {*sm = Some(sampling_method);}- self.context.reset_command_buffer();-// Wait on previous pass if this is a continuationif is_continuation {self.context.command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);@@ -496,13 +495,13 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.executables- .encode(&mut state, &EncodingParameters::new(false, false, false), &mut self.context.command_buffer)+ .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;// Encode samplingself.context.gpu_sampler- .encode(&mut state, &EncodingParameters::new(false, false, false), &mut self.context.command_buffer)+ .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;// Copy sampled token: sampling_output → token_ids (for next pass)@@ -515,19 +514,17 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let token_ids_buf_rc = token_ids_binding.buffer();let mut token_ids_buf_borrow = token_ids_buf_rc.borrow_mut();- self.context.command_buffer.with_compute_encoder(|encoder| {- self.context.token_copy_sampled.encode(- sampling_output_buf_borrow.deref(),- token_ids_buf_borrow.deref_mut(),- encoder,- );- let results_offset = slot * std::mem::size_of::<u32>();- self.context.token_copy_results.encode(- sampling_output_buf_borrow.deref(),- (results_buffer.borrow_mut().deref_mut(), results_offset),- encoder,- );- });+ self.context.token_copy_sampled.encode(+ sampling_output_buf_borrow.deref(),+ token_ids_buf_borrow.deref_mut(),+ &mut self.context.command_buffer,+ );+ let results_offset = slot * std::mem::size_of::<u32>();+ self.context.token_copy_results.encode(+ sampling_output_buf_borrow.deref(),+ (results_buffer.borrow_mut().deref_mut(), results_offset),+ &mut self.context.command_buffer,+ );// Scatter + register for all transformer layersself.context.cache_layers.borrow_mut().update_after_acceptance(@@ -538,26 +535,24 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {);self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);- self.context.command_buffer.with_compute_encoder(|encoder| {- if let Some(mask_update) = &self.context.mask_update {- let updates: Vec<AttentionBiasUpdate> =- self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);- for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {- if let Some(update) = updates.iter().find(|u| &u.key == window_size) {- if update.unmask_col >= 0 || update.mask_col >= 0 {- let mask_buf_rc = mask_buffer.borrow().buffer();- let mut mask_buf_borrow = mask_buf_rc.borrow_mut();- mask_update.encode(- mask_buf_borrow.deref_mut(),- update.unmask_col,- update.mask_col,- encoder,- );- }+ if let Some(mask_update) = &self.context.mask_update {+ let updates: Vec<AttentionBiasUpdate> =+ self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);+ for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {+ if let Some(update) = updates.iter().find(|u| &u.key == window_size) {+ if update.unmask_col >= 0 || update.mask_col >= 0 {+ let mask_buf_rc = mask_buffer.borrow().buffer();+ let mut mask_buf_borrow = mask_buf_rc.borrow_mut();+ mask_update.encode(+ mask_buf_borrow.deref_mut(),+ update.unmask_col,+ update.mask_col,+ &mut self.context.command_buffer,+ );}}}- });+ }// Signal event for next passlet next_counter = current_counter + 1;@@ -567,17 +562,18 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// Add completion handlerlet results_buffer_ptr = results_buffer.borrow().cpu_ptr().as_ptr() as *const u32;- let handler = move |result: Result<(), B::Error>| {+ let handler = move |result: Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>| {result.expect("async decoding forward pass completed with error");let token = { unsafe { *results_buffer_ptr.add(slot) as u64 } };on_complete(token);};self.context.command_buffer.add_completion_handler(handler);- self.context.command_buffer.submit();+ drop(token_ids_binding);+ let pending = self.context.replace_command_buffer().end_encoding().submit();if should_capture {- self.context.command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;}@@ -772,12 +768,12 @@ impl<B: Backend> LanguageModelGenerator<B> {if let Some(_) = self.encoded_tasks.remove(&encoded_task_key) {// Nothing} else {- self.context.reset_command_buffer();+ self.context.replace_command_buffer();task.build_encoded_task(&mut self.context,&mut state,- &EncodingParameters::new(warmup, true, false),+ &EncodingParameters::new(),encoded_task_key.clone(),).map_err(|e| Error::EncodeFailed(Box::new(e)))?;@@ -787,41 +783,34 @@ impl<B: Backend> LanguageModelGenerator<B> {if !task.is_prefilling {self.context.gpu_sampler- .encode(- &mut state,- &EncodingParameters::new(warmup, true, false),- &mut self.context.command_buffer,- )+ .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;}}- self.context.command_buffer.submit();+ let command_buffer = std::mem::replace(+ &mut self.context.command_buffer,+ self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),+ );+ let pending = command_buffer.end_encoding().submit();if allow_pre_encode {- let in_flight_command_buffer = self.context.reset_command_buffer();-let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);let next_encoded_task = task.build_encoded_task(&mut self.context,&mut state,- &EncodingParameters::new(warmup, false, false).with_projection(1),+ &EncodingParameters::new().with_projection(1),next_task_key.clone(),).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);-- in_flight_command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;- } else {- self.context- .command_buffer- .wait_until_completed()- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;}+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+let run_time = run_start.elapsed().as_secs_f64();if should_capture {self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;@@ -857,7 +846,8 @@ impl<B: Backend> LanguageModelGenerator<B> {suffix_start: Option<usize>,wait_until_completed: bool,) -> Result<(), Error> {- let mut command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");+ let mut command_buffer =+ self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();{let mut cache_layers = self.context.cache_layers.borrow_mut();@@ -869,10 +859,10 @@ impl<B: Backend> LanguageModelGenerator<B> {);}- command_buffer.submit();+ let pending = command_buffer.end_encoding().submit();if wait_until_completed {- command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;}Ok(())}diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex b5d576b0d..f473fea09 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -9,7 +9,7 @@ use std::{use crate::{DataType,backends::common::{- Backend, Context, Kernels, NativeBuffer,+ Backend, Buffer, CommandBuffer, CommandBufferInitial, Context, Kernels,kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},},config::{DecoderConfig, LanguageModelConfig, ModelMetadata},@@ -31,13 +31,13 @@ use crate::{pub struct AsyncBuffers<B: Backend> {/// Positions buffer: [max_tokens] i32/// Pre-populated with [prefill_count, prefill_count+1, ...]- pub positions: Rc<RefCell<B::NativeBuffer>>,+ pub positions: Rc<RefCell<B::Buffer>>,/// Seeds buffer: [max_tokens] u64/// Pre-populated with deterministic seed sequence- pub seeds: Rc<RefCell<B::NativeBuffer>>,+ pub seeds: Rc<RefCell<B::Buffer>>,/// Results buffer: [batch_size] u32/// Each pass writes its sampled token to results[pass_idx % batch_size]- pub results: Rc<RefCell<B::NativeBuffer>>,+ pub results: Rc<RefCell<B::Buffer>>,/// Event for GPU-side synchronization between passespub event: B::Event,/// Current event counter (pass N waits on N, signals N+1)@@ -122,7 +122,7 @@ impl<B: Backend> AsyncBuffers<B> {pub struct LanguageModelGeneratorContext<B: Backend> {pub context: Rc<B::Context>,- pub command_buffer: B::CommandBuffer,+ pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,pub cache_layers: Rc<RefCell<CacheLayers<B>>>,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,@@ -151,7 +151,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {decoding_config: &DecodingConfig,) -> Result<Self, Error> {let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();let config_path = model_path.join("config.json");if !config_path.exists() {@@ -247,10 +247,10 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {Ok(context)}- pub fn reset_command_buffer(&mut self) -> B::CommandBuffer {+ pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {std::mem::replace(&mut self.command_buffer,- self.context.create_command_buffer().expect("Failed to create command buffer"),+ self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),)}}diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex 5063c8570..3df98449f 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -2,18 +2,18 @@ use std::mem::size_of;use super::language_model_generator_context::LanguageModelGeneratorContext;use crate::{- backends::common::{Backend, Context, NativeBuffer},+ backends::common::{Backend, Buffer, Context},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::ForwardPassState,};pub struct LanguageModelGeneratorEncodedTask<B: Backend> {pub key: String,- predicate_buffer: B::NativeBuffer,+ predicate_buffer: B::Buffer,}impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {- pub fn predicate_buffer(&self) -> &B::NativeBuffer {+ pub fn predicate_buffer(&self) -> &B::Buffer {&self.predicate_buffer}@@ -111,7 +111,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {&self,context: &mut LanguageModelGeneratorContext<B>,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters<B>,+ parameters: &EncodingParameters,key: String,) -> Result<LanguageModelGeneratorEncodedTask<B>, B::Error> {context.executables.encode(state, parameters, &mut context.command_buffer)?;diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 164c62994..027b25f73 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -18,7 +18,10 @@ use num_traits::NumCast;use crate::{ArrayElement, DataType,array::Array,- backends::common::{Backend, CommandBuffer, Context, kernel::kv_cache_update::KVCacheUpdate},+ backends::common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ kernel::kv_cache_update::KVCacheUpdate,+ },classifier::Classifier,config::ModelMetadata,encodable_block::{EncodableBlock, EncodingParameters, Sampling},@@ -270,13 +273,14 @@ impl<B: Backend> TraceValidator<B> {None,);- let mut command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");+ let mut command_buffer =+ ctx.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();ctx.executables- .encode(&mut state, &EncodingParameters::new(false, false, false), &mut command_buffer)+ .encode(&mut state, &EncodingParameters::new(), &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- command_buffer.submit();- command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ let pending = command_buffer.end_encoding().submit();+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;let traces = state.traces().clone();let data_type = ctx.model_shape.activation_data_type();diff --git a/crates/uzu/tests/allocator_test/allocation_stats.rs b/crates/uzu/tests/allocator_test/allocation_stats.rsdeleted file mode 100644index cf151c835..000000000--- a/crates/uzu/tests/allocator_test/allocation_stats.rs+++ /dev/null@@ -1,46 +0,0 @@-use std::time::Duration;--#[derive(Clone, Copy, Default)]-pub struct AllocationStats {- pub total_allocs: usize,- pub total_frees: usize,- pub total_alloc_time_ns: u128,- pub total_free_time_ns: u128,- pub total_bytes_requested: usize,-}--impl AllocationStats {- pub fn record_alloc(- &mut self,- size: usize,- duration: Duration,- ) {- self.total_allocs += 1;- self.total_alloc_time_ns += duration.as_nanos();- self.total_bytes_requested += size;- }-- pub fn record_free(- &mut self,- duration: Duration,- ) {- self.total_frees += 1;- self.total_free_time_ns += duration.as_nanos();- }-- pub fn avg_alloc_time_us(&self) -> f64 {- if self.total_allocs == 0 {- 0.0- } else {- self.total_alloc_time_ns as f64 / self.total_allocs as f64 / 1000.0- }- }-- pub fn avg_free_time_us(&self) -> f64 {- if self.total_frees == 0 {- 0.0- } else {- self.total_free_time_ns as f64 / self.total_frees as f64 / 1000.0- }- }-}diff --git a/crates/uzu/tests/allocator_test/allocator_trait.rs b/crates/uzu/tests/allocator_test/allocator_trait.rsdeleted file mode 100644index db5eb325d..000000000--- a/crates/uzu/tests/allocator_test/allocator_trait.rs+++ /dev/null@@ -1,15 +0,0 @@-pub trait AllocatorTrait {- type Buffer;-- fn alloc(- &self,- size: usize,- ) -> Self::Buffer;- fn free(- &self,- buffer: Self::Buffer,- );- fn peak_memory(&self) -> usize;- fn cache_memory(&self) -> usize;- fn active_memory(&self) -> usize;-}diff --git a/crates/uzu/tests/allocator_test/caching_allocator.rs b/crates/uzu/tests/allocator_test/caching_allocator.rsdeleted file mode 100644index 31773c871..000000000--- a/crates/uzu/tests/allocator_test/caching_allocator.rs+++ /dev/null@@ -1,51 +0,0 @@-use std::rc::Rc;--use uzu::backends::{- common::{Backend, Buffer, BufferLifetime, Context},- metal::Metal,-};--use super::allocator_trait::AllocatorTrait;--pub struct CachingAllocator {- pub context: Rc<<Metal as Backend>::Context>,-}--impl CachingAllocator {- pub fn new() -> Self {- let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");- Self {- context,- }- }-}--impl AllocatorTrait for CachingAllocator {- type Buffer = Buffer<Metal>;-- fn alloc(- &self,- size: usize,- ) -> Self::Buffer {- self.context.allocator().alloc(BufferLifetime::Scratch, size).expect("Allocation failed")- }-- fn free(- &self,- buffer: Self::Buffer,- ) {- drop(buffer);- }-- fn peak_memory(&self) -> usize {- self.context.allocator().peak_memory()- }-- fn cache_memory(&self) -> usize {- self.context.allocator().cache_memory()- }-- fn active_memory(&self) -> usize {- self.context.allocator().active_memory()- }-}diff --git a/crates/uzu/tests/allocator_test/device_allocator.rs b/crates/uzu/tests/allocator_test/device_allocator.rsdeleted file mode 100644index 0ee9a33ac..000000000--- a/crates/uzu/tests/allocator_test/device_allocator.rs+++ /dev/null@@ -1,67 +0,0 @@-use std::cell::Cell;--use metal::{MTLBuffer, MTLDevice, MTLDeviceExt, MTLResourceOptions};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::allocator_trait::AllocatorTrait;--pub struct DeviceAllocator {- device: Retained<ProtocolObject<dyn MTLDevice>>,- active_memory: Cell<usize>,- peak_memory: Cell<usize>,-}--impl DeviceAllocator {- pub fn new() -> Self {- let device = <dyn MTLDevice>::system_default().expect("No Metal device");-- Self {- device,- active_memory: Cell::new(0),- peak_memory: Cell::new(0),- }- }-}--impl AllocatorTrait for DeviceAllocator {- type Buffer = Retained<ProtocolObject<dyn MTLBuffer>>;-- fn alloc(- &self,- size: usize,- ) -> Self::Buffer {- let buffer =- self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");-- let buf_size = buffer.length();- let new_active = self.active_memory.get() + buf_size;- self.active_memory.set(new_active);-- if new_active > self.peak_memory.get() {- self.peak_memory.set(new_active);- }-- buffer- }-- fn free(- &self,- buffer: Self::Buffer,- ) {- let size = buffer.length();- self.active_memory.set(self.active_memory.get().saturating_sub(size));- drop(buffer);- }-- fn peak_memory(&self) -> usize {- self.peak_memory.get()- }-- fn cache_memory(&self) -> usize {- 0- }-- fn active_memory(&self) -> usize {- self.active_memory.get()- }-}diff --git a/crates/uzu/tests/allocator_test/heap_allocator.rs b/crates/uzu/tests/allocator_test/heap_allocator.rsdeleted file mode 100644index 9376baaae..000000000--- a/crates/uzu/tests/allocator_test/heap_allocator.rs+++ /dev/null@@ -1,76 +0,0 @@-use std::cell::Cell;--use metal::{MTLBuffer, MTLDevice, MTLDeviceExt, MTLHeap, MTLHeapDescriptor, MTLResourceOptions, MTLStorageMode};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::allocator_trait::AllocatorTrait;--pub struct MTLHeapAllocator {- heap: Retained<ProtocolObject<dyn MTLHeap>>,- peak_used: Cell<usize>,- active_memory: Cell<usize>,-}--impl MTLHeapAllocator {- pub fn new(heap_size: usize) -> Self {- let device = <dyn MTLDevice>::system_default().expect("No Metal device");-- let descriptor = MTLHeapDescriptor::new();- descriptor.set_size(heap_size);- descriptor.set_storage_mode(MTLStorageMode::Shared);-- let heap = device.new_heap_with_descriptor(&descriptor).expect("Failed to create heap");-- Self {- heap,- peak_used: Cell::new(0),- active_memory: Cell::new(0),- }- }-}--impl AllocatorTrait for MTLHeapAllocator {- type Buffer = Retained<ProtocolObject<dyn MTLBuffer>>;-- fn alloc(- &self,- size: usize,- ) -> Self::Buffer {- let buffer = self- .heap- .new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer from heap");-- let buf_size = buffer.length();- let new_active = self.active_memory.get() + buf_size;- self.active_memory.set(new_active);-- let used = self.heap.used_size();- if used > self.peak_used.get() {- self.peak_used.set(used);- }-- buffer- }-- fn free(- &self,- buffer: Self::Buffer,- ) {- let size = buffer.length();- self.active_memory.set(self.active_memory.get().saturating_sub(size));- drop(buffer);- }-- fn peak_memory(&self) -> usize {- self.peak_used.get()- }-- fn cache_memory(&self) -> usize {- 0- }-- fn active_memory(&self) -> usize {- self.active_memory.get()- }-}diff --git a/crates/uzu/tests/allocator_test/inference_simulation.rs b/crates/uzu/tests/allocator_test/inference_simulation.rsdeleted file mode 100644index 812108b1e..000000000--- a/crates/uzu/tests/allocator_test/inference_simulation.rs+++ /dev/null@@ -1,197 +0,0 @@-use std::time::Instant;--use super::{allocation_stats::AllocationStats, allocator_trait::AllocatorTrait, model_config::ModelConfig};--const F16_SIZE: usize = 2;--#[derive(Clone, Copy, Debug)]-pub struct MemorySnapshot {- pub step: usize,- pub active_memory: usize,- pub cache_memory: usize,- pub peak_memory: usize,-}--pub struct InferenceSimulation<'a, A: AllocatorTrait> {- allocator: &'a A,- config: ModelConfig,- pub stats: AllocationStats,- pub current_prefix_len: usize,- step_counter: usize,-}--impl<'a, A: AllocatorTrait> InferenceSimulation<'a, A> {- pub fn new(- allocator: &'a A,- config: ModelConfig,- ) -> Self {- Self {- allocator,- config,- stats: AllocationStats::default(),- current_prefix_len: 0,- step_counter: 0,- }- }-- pub fn capture_snapshot(&self) -> MemorySnapshot {- MemorySnapshot {- step: self.step_counter,- active_memory: self.allocator.active_memory(),- cache_memory: self.allocator.cache_memory(),- peak_memory: self.allocator.peak_memory(),- }- }-- fn alloc_scratch(- &mut self,- size: usize,- ) -> A::Buffer {- let start = Instant::now();- let result = self.allocator.alloc(size);- self.stats.record_alloc(size, start.elapsed());- result- }-- fn free(- &mut self,- buffer: A::Buffer,- ) {- let start = Instant::now();- self.allocator.free(buffer);- self.stats.record_free(start.elapsed());- }-- fn simulate_embedding_kernel(- &mut self,- batch_size: usize,- ) -> A::Buffer {- let size = batch_size * self.config.model_dim * F16_SIZE;- self.alloc_scratch(size)- }-- fn simulate_attention_kernel(- &mut self,- batch_size: usize,- ) -> Vec<A::Buffer> {- let mut buffers = Vec::new();-- let qkv_size = batch_size * self.config.qkv_dim() * F16_SIZE;- buffers.push(self.alloc_scratch(qkv_size));-- let bias_size = batch_size * (batch_size + self.current_prefix_len) * F16_SIZE;- buffers.push(self.alloc_scratch(bias_size));-- let rotated_q_size = self.config.num_heads * batch_size * self.config.head_dim * F16_SIZE;- buffers.push(self.alloc_scratch(rotated_q_size));-- let rotated_k_size = self.config.num_groups * batch_size * self.config.head_dim * F16_SIZE;- buffers.push(self.alloc_scratch(rotated_k_size));-- let attn_out_size = batch_size * self.config.num_heads * self.config.head_dim * F16_SIZE;- buffers.push(self.alloc_scratch(attn_out_size));-- buffers- }-- fn simulate_mlp_kernel(- &mut self,- batch_size: usize,- ) -> Vec<A::Buffer> {- let mut buffers = Vec::new();-- let up_size = batch_size * 2 * self.config.hidden_dim * F16_SIZE;- buffers.push(self.alloc_scratch(up_size));-- let hidden_size = batch_size * self.config.hidden_dim * F16_SIZE;- buffers.push(self.alloc_scratch(hidden_size));-- buffers- }-- fn simulate_logits_kernel(- &mut self,- batch_size: usize,- ) -> A::Buffer {- let size = batch_size * self.config.vocab_size * F16_SIZE;- self.alloc_scratch(size)- }-- fn simulate_forward_pass(- &mut self,- batch_size: usize,- ) {- self.step_counter += 1;-- let main = self.simulate_embedding_kernel(batch_size);-- for _layer in 0..self.config.num_layers {- let attn_buffers = self.simulate_attention_kernel(batch_size);- for buf in attn_buffers {- self.free(buf);- }-- let mlp_buffers = self.simulate_mlp_kernel(batch_size);- for buf in mlp_buffers {- self.free(buf);- }- }-- let logits = self.simulate_logits_kernel(batch_size);- self.free(logits);- self.free(main);- }-- pub fn run_prefill(- &mut self,- prompt_length: usize,- step_size: usize,- ) -> Vec<MemorySnapshot> {- let mut snapshots = Vec::new();- let mut remaining = prompt_length;-- while remaining > 0 {- let batch_size = remaining.min(step_size);- self.simulate_forward_pass(batch_size);- self.current_prefix_len += batch_size;- remaining = remaining.saturating_sub(step_size);- snapshots.push(self.capture_snapshot());- }-- snapshots- }-- pub fn run_generation(- &mut self,- num_tokens: usize,- ) -> Vec<MemorySnapshot> {- let mut snapshots = Vec::new();-- for _ in 0..num_tokens {- self.simulate_forward_pass(1);- self.current_prefix_len += 1;- snapshots.push(self.capture_snapshot());- }-- snapshots- }-- pub fn run_generation_with_sampling(- &mut self,- num_tokens: usize,- snapshot_interval: usize,- ) -> Vec<MemorySnapshot> {- let mut snapshots = Vec::new();-- for i in 0..num_tokens {- self.simulate_forward_pass(1);- self.current_prefix_len += 1;-- if i % snapshot_interval == 0 || i == num_tokens - 1 {- snapshots.push(self.capture_snapshot());- }- }-- snapshots- }-}diff --git a/crates/uzu/tests/allocator_test/main.rs b/crates/uzu/tests/allocator_test/main.rsdeleted file mode 100644index 0d0741726..000000000--- a/crates/uzu/tests/allocator_test/main.rs+++ /dev/null@@ -1,270 +0,0 @@-#![cfg(target_os = "macos")]--mod allocation_stats;-mod allocator_trait;-mod caching_allocator;-mod device_allocator;-mod heap_allocator;-mod inference_simulation;-mod model_config;-mod simulation_result;--use caching_allocator::CachingAllocator;-use device_allocator::DeviceAllocator;-use heap_allocator::MTLHeapAllocator;-use model_config::ModelConfig;-use simulation_result::{run_long_simulation, run_simulation, run_simulation_with_config};-use uzu::backends::common::Context;--const KB: f64 = 1024.0;-const MB: f64 = 1024.0 * 1024.0;--fn print_comparison(- name: &str,- uzu_result: &simulation_result::SimulationResult,- heap_result: &simulation_result::SimulationResult,- device_result: &simulation_result::SimulationResult,-) {- println!("\n=== {} ===", name);- println!("{:<20} {:>14} {:>14} {:>14}", "Metric", "UZU Caching", "Heap", "Device");- println!("{:-<62}", "");-- println!(- "{:<20} {:>14} {:>14} {:>14}",- "Allocations", uzu_result.stats.total_allocs, heap_result.stats.total_allocs, device_result.stats.total_allocs- );-- println!(- "{:<20} {:>14} {:>14} {:>14}",- "Frees", uzu_result.stats.total_frees, heap_result.stats.total_frees, device_result.stats.total_frees- );-- println!(- "{:<20} {:>13.2}µs {:>13.2}µs {:>13.2}µs",- "Avg alloc time",- uzu_result.stats.avg_alloc_time_us(),- heap_result.stats.avg_alloc_time_us(),- device_result.stats.avg_alloc_time_us()- );-- println!(- "{:<20} {:>13.2}µs {:>13.2}µs {:>13.2}µs",- "Avg free time",- uzu_result.stats.avg_free_time_us(),- heap_result.stats.avg_free_time_us(),- device_result.stats.avg_free_time_us()- );-- println!(- "{:<20} {:>12.2} MB {:>12.2} MB {:>12.2} MB",- "Bytes requested",- uzu_result.stats.total_bytes_requested as f64 / MB,- heap_result.stats.total_bytes_requested as f64 / MB,- device_result.stats.total_bytes_requested as f64 / MB- );-- println!(- "{:<20} {:>12.2} MB {:>12.2} MB {:>12.2} MB",- "Peak memory",- uzu_result.peak_memory_bytes as f64 / MB,- heap_result.peak_memory_bytes as f64 / MB,- device_result.peak_memory_bytes as f64 / MB- );-- println!(- "{:<20} {:>12.2} KB {:>12.2} KB {:>12.2} KB",- "Final cache",- uzu_result.cache_memory_bytes as f64 / KB,- heap_result.cache_memory_bytes as f64 / KB,- device_result.cache_memory_bytes as f64 / KB- );-- println!();-- let device_time = device_result.stats.avg_alloc_time_us().max(0.001);- let uzu_speedup = device_time / uzu_result.stats.avg_alloc_time_us().max(0.001);- let heap_speedup = device_time / heap_result.stats.avg_alloc_time_us().max(0.001);-- println!("Alloc speedup vs device: UZU {:.1}x, Heap {:.1}x", uzu_speedup, heap_speedup);-}--fn print_memory_over_time(- name: &str,- snapshots: &[inference_simulation::MemorySnapshot],- sample_count: usize,-) {- if snapshots.is_empty() {- return;- }-- println!("\n--- {} Memory Over Time (sampled) ---", name);- println!("{:>8} {:>12} {:>12} {:>12}", "Step", "Active", "Cache", "Peak");-- let step = (snapshots.len() / sample_count).max(1);- for (i, snapshot) in snapshots.iter().enumerate() {- if i % step == 0 || i == snapshots.len() - 1 {- println!(- "{:>8} {:>10.2}MB {:>10.2}MB {:>10.2}MB",- snapshot.step,- snapshot.active_memory as f64 / MB,- snapshot.cache_memory as f64 / MB,- snapshot.peak_memory as f64 / MB- );- }- }-}--#[test]-fn test_allocator_comparison() {- let prompt_length = 256;- let generate_length = 2048;- let prefill_step_size = 512;-- println!("\n========================================");- println!(" Short Conversation Test");- println!(" {}p + {}g tokens", prompt_length, generate_length);- println!("========================================");-- let uzu_allocator = CachingAllocator::new();- let uzu_result = run_simulation(&uzu_allocator, prompt_length, generate_length, prefill_step_size);-- let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);- let heap_result = run_simulation(&heap_allocator, prompt_length, generate_length, prefill_step_size);-- let device_allocator = DeviceAllocator::new();- let device_result = run_simulation(&device_allocator, prompt_length, generate_length, prefill_step_size);-- print_comparison("Short Conversation Results", &uzu_result, &heap_result, &device_result);-- assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");- assert_eq!(- heap_result.stats.total_allocs, heap_result.stats.total_frees,- "Heap allocator: allocs should equal frees"- );- assert_eq!(- device_result.stats.total_allocs, device_result.stats.total_frees,- "Device allocator: allocs should equal frees"- );-}--#[test]-fn test_long_conversation_comparison() {- let num_conversations = 10;- let avg_prompt_length = 512;- let avg_generate_length = 1024;- let prefill_step_size = 512;- let snapshot_interval = 100;-- println!("\n========================================");- println!(" Long Conversation Test");- println!(" {} conversations, ~{}p + ~{}g tokens each", num_conversations, avg_prompt_length, avg_generate_length);- println!("========================================");-- let gc_limit = 256 * 1024 * 1024;- let max_pool = 128 * 1024 * 1024;-- let uzu_allocator = CachingAllocator::new();- uzu_allocator.context.allocator().set_gc_limit(gc_limit);- uzu_allocator.context.allocator().set_max_pool_size(max_pool);-- let uzu_result = run_long_simulation(- &uzu_allocator,- num_conversations,- avg_prompt_length,- avg_generate_length,- prefill_step_size,- snapshot_interval,- );-- let heap_allocator = MTLHeapAllocator::new(1024 * 1024 * 1024);- let heap_result = run_long_simulation(- &heap_allocator,- num_conversations,- avg_prompt_length,- avg_generate_length,- prefill_step_size,- snapshot_interval,- );-- let device_allocator = DeviceAllocator::new();- let device_result = run_long_simulation(- &device_allocator,- num_conversations,- avg_prompt_length,- avg_generate_length,- prefill_step_size,- snapshot_interval,- );-- print_comparison("Long Conversation Results", &uzu_result, &heap_result, &device_result);-- print_memory_over_time("UZU Caching", &uzu_result.snapshots, 15);- print_memory_over_time("Heap", &heap_result.snapshots, 15);- print_memory_over_time("Device (baseline)", &device_result.snapshots, 15);-- println!("\n--- Memory Efficiency Summary ---");- println!(- "UZU peak/requested ratio: {:.2}%",- (uzu_result.peak_memory_bytes as f64 / uzu_result.stats.total_bytes_requested as f64) * 100.0- );- println!(- "Heap peak/requested ratio: {:.2}%",- (heap_result.peak_memory_bytes as f64 / heap_result.stats.total_bytes_requested as f64) * 100.0- );- println!(- "Device peak/requested ratio: {:.2}%",- (device_result.peak_memory_bytes as f64 / device_result.stats.total_bytes_requested as f64) * 100.0- );-- assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");- assert_eq!(- device_result.stats.total_allocs, device_result.stats.total_frees,- "Device allocator: allocs should equal frees"- );-}--#[test]-fn test_multi_model_comparison() {- let prompt_length = 256;- let generate_length = 512;- let prefill_step_size = 256;-- println!("\n========================================");- println!(" Multi-Model Comparison Test");- println!(" {}p + {}g tokens per model", prompt_length, generate_length);- println!("========================================");-- println!("\n{:<15} {:>12} {:>12} {:>12}", "Model", "UZU (µs)", "Heap (µs)", "Device (µs)");- println!("{:-<55}", "");-- for config in ModelConfig::all_models() {- let uzu_allocator = CachingAllocator::new();- let uzu_result =- run_simulation_with_config(&uzu_allocator, config, prompt_length, generate_length, prefill_step_size);-- let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);- let heap_result =- run_simulation_with_config(&heap_allocator, config, prompt_length, generate_length, prefill_step_size);-- let device_allocator = DeviceAllocator::new();- let device_result =- run_simulation_with_config(&device_allocator, config, prompt_length, generate_length, prefill_step_size);-- println!(- "{:<15} {:>12.2} {:>12.2} {:>12.2}",- config.name,- uzu_result.stats.avg_alloc_time_us(),- heap_result.stats.avg_alloc_time_us(),- device_result.stats.avg_alloc_time_us()- );-- assert_eq!(- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,- "{}: UZU allocs should equal frees",- config.name- );- }-- println!();- println!("Summary: Lower allocation time is better.");-}diff --git a/crates/uzu/tests/allocator_test/mod.rs b/crates/uzu/tests/allocator_test/mod.rsdeleted file mode 100644index 32883483e..000000000--- a/crates/uzu/tests/allocator_test/mod.rs+++ /dev/null@@ -1,279 +0,0 @@-#![cfg(target_os = "macos")]--mod allocation_stats;-mod allocator_trait;-mod caching_allocator;-mod device_allocator;-mod heap_allocator;-mod inference_simulation;-mod mlx_allocator;-mod model_config;-mod simulation_result;--use caching_allocator::CachingAllocator;-use device_allocator::DeviceAllocator;-use heap_allocator::MTLHeapAllocator;-use mlx_allocator::MlxAllocator;-use simulation_result::{run_long_simulation, run_simulation};-use uzu::backends::common::Context;--const KB: f64 = 1024.0;-const MB: f64 = 1024.0 * 1024.0;-const GB: f64 = 1024.0 * 1024.0 * 1024.0;--fn print_three_way_comparison(- name: &str,- uzu_result: &simulation_result::SimulationResult,- mlx_result: &simulation_result::SimulationResult,- heap_result: &simulation_result::SimulationResult,-) {- println!("\n=== {} ===", name);- println!(- "{:<25} {:>15} {:>15} {:>15}",- "Metric", "UZU Caching", "MLX-style LRU", "Heap (baseline)"- );- println!("{:-<72}", "");-- println!(- "{:<25} {:>15} {:>15} {:>15}",- "Allocations",- uzu_result.stats.total_allocs,- mlx_result.stats.total_allocs,- heap_result.stats.total_allocs- );-- println!(- "{:<25} {:>15} {:>15} {:>15}",- "Frees",- uzu_result.stats.total_frees,- mlx_result.stats.total_frees,- heap_result.stats.total_frees- );-- println!(- "{:<25} {:>14.2}µs {:>14.2}µs {:>14.2}µs",- "Avg alloc time",- uzu_result.stats.avg_alloc_time_us(),- mlx_result.stats.avg_alloc_time_us(),- heap_result.stats.avg_alloc_time_us()- );-- println!(- "{:<25} {:>14.2}µs {:>14.2}µs {:>14.2}µs",- "Avg free time",- uzu_result.stats.avg_free_time_us(),- mlx_result.stats.avg_free_time_us(),- heap_result.stats.avg_free_time_us()- );-- println!(- "{:<25} {:>13.2} MB {:>13.2} MB {:>13.2} MB",- "Bytes requested",- uzu_result.stats.total_bytes_requested as f64 / MB,- mlx_result.stats.total_bytes_requested as f64 / MB,- heap_result.stats.total_bytes_requested as f64 / MB- );-- println!(- "{:<25} {:>13.2} MB {:>13.2} MB {:>13.2} MB",- "Peak memory",- uzu_result.peak_memory_bytes as f64 / MB,- mlx_result.peak_memory_bytes as f64 / MB,- heap_result.peak_memory_bytes as f64 / MB- );-- println!(- "{:<25} {:>13.2} KB {:>13.2} KB {:>13.2} KB",- "Final cache",- uzu_result.cache_memory_bytes as f64 / KB,- mlx_result.cache_memory_bytes as f64 / KB,- heap_result.cache_memory_bytes as f64 / KB- );-- println!();-- let uzu_speedup = heap_result.stats.avg_alloc_time_us()- / uzu_result.stats.avg_alloc_time_us().max(0.001);- let mlx_speedup = heap_result.stats.avg_alloc_time_us()- / mlx_result.stats.avg_alloc_time_us().max(0.001);-- println!(- "Alloc speedup vs heap: UZU {:.1}x, MLX {:.1}x",- uzu_speedup, mlx_speedup- );-}--fn print_memory_over_time(- name: &str,- snapshots: &[inference_simulation::MemorySnapshot],- sample_count: usize,-) {- if snapshots.is_empty() {- return;- }-- println!("\n--- {} Memory Over Time (sampled) ---", name);- println!("{:>8} {:>12} {:>12} {:>12}", "Step", "Active", "Cache", "Peak");-- let step = (snapshots.len() / sample_count).max(1);- for (i, snapshot) in snapshots.iter().enumerate() {- if i % step == 0 || i == snapshots.len() - 1 {- println!(- "{:>8} {:>10.2}MB {:>10.2}MB {:>10.2}MB",- snapshot.step,- snapshot.active_memory as f64 / MB,- snapshot.cache_memory as f64 / MB,- snapshot.peak_memory as f64 / MB- );- }- }-}--#[test]-fn test_allocator_comparison() {- let prompt_length = 256;- let generate_length = 2048;- let prefill_step_size = 512;-- println!("\n========================================");- println!(" Short Conversation Test");- println!(" {}p + {}g tokens", prompt_length, generate_length);- println!("========================================");-- let uzu_allocator = CachingAllocator::new();- let uzu_result = run_simulation(- &uzu_allocator,- prompt_length,- generate_length,- prefill_step_size,- );-- let mlx_allocator = MlxAllocator::new(512 * 1024 * 1024, 256 * 1024 * 1024);- let mlx_result = run_simulation(- &mlx_allocator,- prompt_length,- generate_length,- prefill_step_size,- );-- let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);- let heap_result = run_simulation(- &heap_allocator,- prompt_length,- generate_length,- prefill_step_size,- );-- print_three_way_comparison(- "Short Conversation Results",- &uzu_result,- &mlx_result,- &heap_result,- );-- assert_eq!(- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,- "UZU allocator: allocs should equal frees"- );- assert_eq!(- mlx_result.stats.total_allocs, mlx_result.stats.total_frees,- "MLX allocator: allocs should equal frees"- );- assert_eq!(- heap_result.stats.total_allocs, heap_result.stats.total_frees,- "Heap allocator: allocs should equal frees"- );-}--#[test]-fn test_long_conversation_comparison() {- let num_conversations = 10;- let avg_prompt_length = 512;- let avg_generate_length = 1024;- let prefill_step_size = 512;- let snapshot_interval = 100;-- println!("\n========================================");- println!(" Long Conversation Test");- println!(- " {} conversations, ~{}p + ~{}g tokens each",- num_conversations, avg_prompt_length, avg_generate_length- );- println!("========================================");-- let gc_limit = 256 * 1024 * 1024;- let max_pool = 128 * 1024 * 1024;-- let uzu_allocator = CachingAllocator::new();- uzu_allocator.context.allocator().set_gc_limit(gc_limit);- uzu_allocator.context.allocator().set_max_pool_size(max_pool);-- let uzu_result = run_long_simulation(- &uzu_allocator,- num_conversations,- avg_prompt_length,- avg_generate_length,- prefill_step_size,- snapshot_interval,- );-- let mlx_allocator = MlxAllocator::new(gc_limit, max_pool);- let mlx_result = run_long_simulation(- &mlx_allocator,- num_conversations,- avg_prompt_length,- avg_generate_length,- prefill_step_size,- snapshot_interval,- );-- let heap_allocator = MTLHeapAllocator::new(1024 * 1024 * 1024);- let heap_result = run_long_simulation(- &heap_allocator,- num_conversations,- avg_prompt_length,- avg_generate_length,- prefill_step_size,- snapshot_interval,- );-- print_three_way_comparison(- "Long Conversation Results",- &uzu_result,- &mlx_result,- &heap_result,- );-- print_memory_over_time("UZU Caching", &uzu_result.snapshots, 15);- print_memory_over_time("MLX-style LRU", &mlx_result.snapshots, 15);- print_memory_over_time("Heap (baseline)", &heap_result.snapshots, 15);-- println!("\n--- Memory Efficiency Summary ---");- println!(- "UZU peak/requested ratio: {:.2}%",- (uzu_result.peak_memory_bytes as f64- / uzu_result.stats.total_bytes_requested as f64)- * 100.0- );- println!(- "MLX peak/requested ratio: {:.2}%",- (mlx_result.peak_memory_bytes as f64- / mlx_result.stats.total_bytes_requested as f64)- * 100.0- );- println!(- "Heap peak/requested ratio: {:.2}%",- (heap_result.peak_memory_bytes as f64- / heap_result.stats.total_bytes_requested as f64)- * 100.0- );-- assert_eq!(- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,- "UZU allocator: allocs should equal frees"- );- assert_eq!(- mlx_result.stats.total_allocs, mlx_result.stats.total_frees,- "MLX allocator: allocs should equal frees"- );-}diff --git a/crates/uzu/tests/allocator_test/model_config.rs b/crates/uzu/tests/allocator_test/model_config.rsdeleted file mode 100644index f398da479..000000000--- a/crates/uzu/tests/allocator_test/model_config.rs+++ /dev/null@@ -1,106 +0,0 @@-#[derive(Clone, Copy)]-pub struct ModelConfig {- pub name: &'static str,- pub model_dim: usize,- pub num_heads: usize,- pub num_groups: usize,- pub head_dim: usize,- pub hidden_dim: usize,- pub vocab_size: usize,- pub num_layers: usize,-}--impl ModelConfig {- pub fn llama_3_2_1b() -> Self {- Self {- name: "Llama-3.2-1B",- model_dim: 2048,- num_heads: 32,- num_groups: 8,- head_dim: 64,- hidden_dim: 8192,- vocab_size: 128256,- num_layers: 16,- }- }-- pub fn llama_3_2_3b() -> Self {- Self {- name: "Llama-3.2-3B",- model_dim: 3072,- num_heads: 24,- num_groups: 8,- head_dim: 128,- hidden_dim: 8192,- vocab_size: 128256,- num_layers: 28,- }- }-- pub fn qwen3_4b() -> Self {- Self {- name: "Qwen3-4B",- model_dim: 2560,- num_heads: 32,- num_groups: 8,- head_dim: 128,- hidden_dim: 9728,- vocab_size: 151936,- num_layers: 36,- }- }-- pub fn qwen3_8b() -> Self {- Self {- name: "Qwen3-8B",- model_dim: 4096,- num_heads: 32,- num_groups: 8,- head_dim: 128,- hidden_dim: 12288,- vocab_size: 151936,- num_layers: 36,- }- }-- pub fn gemma_3_4b() -> Self {- Self {- name: "Gemma-3-4B",- model_dim: 2560,- num_heads: 8,- num_groups: 4,- head_dim: 256,- hidden_dim: 10240,- vocab_size: 262208,- num_layers: 34,- }- }-- pub fn qwen3_14b() -> Self {- Self {- name: "Qwen3-14B",- model_dim: 5120,- num_heads: 40,- num_groups: 8,- head_dim: 128,- hidden_dim: 17408,- vocab_size: 151936,- num_layers: 40,- }- }-- pub fn all_models() -> Vec<Self> {- vec![- Self::llama_3_2_1b(),- Self::llama_3_2_3b(),- Self::qwen3_4b(),- Self::qwen3_8b(),- Self::gemma_3_4b(),- Self::qwen3_14b(),- ]- }-- pub fn qkv_dim(&self) -> usize {- (2 * self.num_groups + self.num_heads) * self.head_dim- }-}diff --git a/crates/uzu/tests/allocator_test/simulation_result.rs b/crates/uzu/tests/allocator_test/simulation_result.rsdeleted file mode 100644index d3f2a3184..000000000--- a/crates/uzu/tests/allocator_test/simulation_result.rs+++ /dev/null@@ -1,113 +0,0 @@-use super::{- allocation_stats::AllocationStats,- allocator_trait::AllocatorTrait,- inference_simulation::{InferenceSimulation, MemorySnapshot},- model_config::ModelConfig,-};--pub struct SimulationResult {- pub stats: AllocationStats,- pub peak_memory_bytes: usize,- pub cache_memory_bytes: usize,- #[allow(dead_code)]- pub final_prefix_len: usize,- pub snapshots: Vec<MemorySnapshot>,-}--pub fn run_simulation<A: AllocatorTrait>(- allocator: &A,- prompt_length: usize,- generate_length: usize,- prefill_step_size: usize,-) -> SimulationResult {- let config = ModelConfig::llama_3_2_1b();- let mut sim = InferenceSimulation::new(allocator, config);-- let mut snapshots = sim.run_prefill(prompt_length, prefill_step_size);- let gen_snapshots = sim.run_generation(generate_length);- snapshots.extend(gen_snapshots);-- SimulationResult {- stats: sim.stats,- peak_memory_bytes: allocator.peak_memory(),- cache_memory_bytes: allocator.cache_memory(),- final_prefix_len: sim.current_prefix_len,- snapshots,- }-}--pub fn run_long_simulation<A: AllocatorTrait>(- allocator: &A,- num_conversations: usize,- avg_prompt_length: usize,- avg_generate_length: usize,- prefill_step_size: usize,- snapshot_interval: usize,-) -> SimulationResult {- let config = ModelConfig::llama_3_2_1b();- run_long_simulation_with_config(- allocator,- config,- num_conversations,- avg_prompt_length,- avg_generate_length,- prefill_step_size,- snapshot_interval,- )-}--pub fn run_simulation_with_config<A: AllocatorTrait>(- allocator: &A,- config: ModelConfig,- prompt_length: usize,- generate_length: usize,- prefill_step_size: usize,-) -> SimulationResult {- let mut sim = InferenceSimulation::new(allocator, config);-- let mut snapshots = sim.run_prefill(prompt_length, prefill_step_size);- let gen_snapshots = sim.run_generation(generate_length);- snapshots.extend(gen_snapshots);-- SimulationResult {- stats: sim.stats,- peak_memory_bytes: allocator.peak_memory(),- cache_memory_bytes: allocator.cache_memory(),- final_prefix_len: sim.current_prefix_len,- snapshots,- }-}--pub fn run_long_simulation_with_config<A: AllocatorTrait>(- allocator: &A,- config: ModelConfig,- num_conversations: usize,- avg_prompt_length: usize,- avg_generate_length: usize,- prefill_step_size: usize,- snapshot_interval: usize,-) -> SimulationResult {- let mut sim = InferenceSimulation::new(allocator, config);- let mut all_snapshots = Vec::new();-- for conv_idx in 0..num_conversations {- let prompt_len = avg_prompt_length + (conv_idx % 3) * (avg_prompt_length / 4);- let gen_len = avg_generate_length + (conv_idx % 5) * (avg_generate_length / 8);-- let prefill_snapshots = sim.run_prefill(prompt_len, prefill_step_size);- all_snapshots.extend(prefill_snapshots);-- let gen_snapshots = sim.run_generation_with_sampling(gen_len, snapshot_interval);- all_snapshots.extend(gen_snapshots);-- sim.current_prefix_len = 0;- }-- SimulationResult {- stats: sim.stats,- peak_memory_bytes: allocator.peak_memory(),- cache_memory_bytes: allocator.cache_memory(),- final_prefix_len: sim.current_prefix_len,- snapshots: all_snapshots,- }-}diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex d53cf4c76..5e52c29db 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -5,14 +5,15 @@ mod common;use std::mem::size_of;use bytemuck;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use ndarray::{Array3, Array4, s};use objc2::{rc::Retained, runtime::ProtocolObject};use uzu::{DataType,backends::{common::{- Backend, CommandBuffer, Context, Kernels,+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending, Context, Kernels,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,attention::{AttentionGemmArguments, AttentionGemmBlock},@@ -316,8 +317,7 @@ fn run_single_pass_attention(.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -347,12 +347,9 @@ fn run_single_pass_attention(sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,- &mut compute_encoder,+ &mut command_buffer,);- compute_encoder.end_encoding();-- command_buffer.commit();- command_buffer.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -389,8 +386,7 @@ fn run_single_pass_attention_with_is_causal(.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -419,12 +415,9 @@ fn run_single_pass_attention_with_is_causal(sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,- &mut compute_encoder,+ &mut command_buffer,);- compute_encoder.end_encoding();-- command_buffer.commit();- command_buffer.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -460,8 +453,7 @@ fn run_gemm_attention(.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();let args = AttentionGemmArguments {queries_buffer: &query_buffer,@@ -481,12 +473,8 @@ fn run_gemm_attention(scale,};- let encode_result = kernel.encode(context, &mut compute_encoder, args);- compute_encoder.end_encoding();- encode_result?;-- command_buffer.commit();- command_buffer.wait_until_completed().unwrap();+ kernel.encode(context, &mut command_buffer, args)?;+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -935,8 +923,7 @@ fn run_two_pass_attention().expect("Failed to create buffer");- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -967,7 +954,7 @@ fn run_two_pass_attention(mask_q_seq_stride,mask_head_stride,sinks_buffer.as_ref().map(|b| b),- &mut compute_encoder,+ &mut command_buffer,);kernel_pass2.encode(&partials_buffer,@@ -976,12 +963,9 @@ fn run_two_pass_attention(&mut output_buffer,num_heads as u32,seq_len as u32,- &mut compute_encoder,+ &mut command_buffer,);- compute_encoder.end_encoding();-- command_buffer.commit();- command_buffer.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -1173,8 +1157,7 @@ fn perf_two_pass_attention() {.expect("Failed to create buffer");// ---- Launch and time ----- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;@@ -1199,7 +1182,7 @@ fn perf_two_pass_attention() {None,None,sinks_buffer.as_ref().map(|b| b),- &mut compute_encoder,+ &mut command_buffer,);kernel_pass2.encode(&partials_buffer,@@ -1208,18 +1191,15 @@ fn perf_two_pass_attention() {&mut output_buffer,num_heads as u32,suffix_length as u32,- &mut compute_encoder,+ &mut command_buffer,);- compute_encoder.end_encoding();-// Time both host-side and GPU executionlet host_timer = Instant::now();- command_buffer.commit();- command_buffer.wait_until_completed().unwrap();+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;// Get actual GPU execution time- let gpu_elapsed_ms = command_buffer.gpu_execution_time_ms();+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();match gpu_elapsed_ms {Some(gpu_time) => {diff --git a/crates/uzu/tests/kernel/full_precision_embedding_test.rs b/crates/uzu/tests/kernel/full_precision_embedding_test.rsindex ee634ba65..2a1b61fd8 100644--- a/crates/uzu/tests/kernel/full_precision_embedding_test.rs+++ b/crates/uzu/tests/kernel/full_precision_embedding_test.rs@@ -8,7 +8,10 @@ use num_traits::Float;use uzu::{ArrayElement,array::ArrayContextExt,- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::FullPrecisionEmbeddingLookupKernel},+ backends::common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Kernels, kernel::FullPrecisionEmbeddingLookupKernel,+ },};struct Input<T: ArrayElement + Float> {@@ -60,21 +63,18 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights, "");let output_array = context.create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to get command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(- token_ids_array.buffer().borrow().deref(),- weights_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),- input.batch_size as u32,- input.vocab_size as u32,- input.model_dim as u32,- input.input_scale,- encoder,- );- });- command_buffer.submit();- command_buffer.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to get command buffer").start_encoding();+ kernel.encode(+ token_ids_array.buffer().borrow().deref(),+ weights_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size as u32,+ input.vocab_size as u32,+ input.model_dim as u32,+ input.input_scale,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/kernel/qk_norm_test.rs b/crates/uzu/tests/kernel/qk_norm_test.rsindex 562f390fb..b587efecc 100644--- a/crates/uzu/tests/kernel/qk_norm_test.rs+++ b/crates/uzu/tests/kernel/qk_norm_test.rs@@ -9,7 +9,10 @@ use uzu::{ArrayElement, DataType,array::ArrayContextExt,backends::{- common::{Backend, CommandBuffer, Context, Kernels, kernel::QKNormKernel},+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::QKNormKernel,+ },cpu::Cpu,},};@@ -130,29 +133,24 @@ fn get_output<let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(- None::<&B::NativeBuffer>,- scales_array.buffer().borrow().deref(),- qkv_array.buffer().borrow_mut().deref_mut(),- input.batch_size,- input.num_q_heads,- input.num_kv_heads,- input.head_dim,- input.epsilon,- input.scale_offset,- input.head_offset,- input.head_count,- input.full_layer,- encoder,- )- });-- command_buffer.submit();- if let Err(err) = command_buffer.wait_until_completed() {- panic!("Failed to wait command buffer: {}", err);- }+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ None::<&B::Buffer>,+ scales_array.buffer().borrow().deref(),+ qkv_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.num_q_heads,+ input.num_kv_heads,+ input.head_dim,+ input.epsilon,+ input.scale_offset,+ input.head_offset,+ input.head_count,+ input.full_layer,+ &mut command_buffer,+ );++ command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");qkv_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/kernel/rms_norm_test.rs b/crates/uzu/tests/kernel/rms_norm_test.rsindex dd6df922a..3b13ab441 100644--- a/crates/uzu/tests/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/kernel/rms_norm_test.rs@@ -13,7 +13,10 @@ use uzu::{ArrayElement, DataType,array::ArrayContextExt,backends::{- common::{Backend, CommandBuffer, Context, Kernels, kernel::RMSNormKernel},+ common::{+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending, Context, Kernels, kernel::RMSNormKernel,+ },cpu::Cpu,},};@@ -145,28 +148,24 @@ fn get_output<false => context.create_array_uninitialized(&[input_size], OutputT::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(- input_buffer,- scales_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),- input.batch_size,- input.element_count,- input.epsilon,- input.scale_offset,- input.full_layer,- encoder,- )- });+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ input_buffer,+ scales_array.buffer().borrow().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.batch_size,+ input.element_count,+ input.epsilon,+ input.scale_offset,+ input.full_layer,+ &mut command_buffer,+ );let instant = Instant::now();- command_buffer.submit();- if let Err(err) = command_buffer.wait_until_completed() {- panic!("Failed to wait command buffer: {}", err);- }+ let completed =+ command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let host_elapsed_ms = instant.elapsed().as_secs_f64() * 1e3;- let gpu_elapsed_ms = command_buffer.gpu_execution_time_ms();+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();(output_array.as_slice().to_vec(), host_elapsed_ms, gpu_elapsed_ms)}diff --git a/crates/uzu/tests/kernel/tensor_add_bias_test.rs b/crates/uzu/tests/kernel/tensor_add_bias_test.rsindex 7014f6db1..363849284 100644--- a/crates/uzu/tests/kernel/tensor_add_bias_test.rs+++ b/crates/uzu/tests/kernel/tensor_add_bias_test.rs@@ -8,7 +8,10 @@ use num_traits::Float;use uzu::{ArrayElement,array::ArrayContextExt,- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddBiasKernel},+ backends::common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Kernels, kernel::TensorAddBiasKernel,+ },};struct Input<T: ArrayElement + Float> {@@ -75,19 +78,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(false => context.create_array_uninitialized(&[size], T::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(- input_buffer,- bias_array.buffer().borrow_mut().deref(),- output_array.buffer().borrow_mut().deref_mut(),- input.num_cols,- input.length,- encoder,- )- });- command_buffer.submit();- command_buffer.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ input_buffer,+ bias_array.buffer().borrow_mut().deref(),+ output_array.buffer().borrow_mut().deref_mut(),+ input.num_cols,+ input.length,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/kernel/tensor_add_swap_test.rs b/crates/uzu/tests/kernel/tensor_add_swap_test.rsindex d6c8b1952..cd5926cc8 100644--- a/crates/uzu/tests/kernel/tensor_add_swap_test.rs+++ b/crates/uzu/tests/kernel/tensor_add_swap_test.rs@@ -5,7 +5,10 @@ use num_traits::Float;use uzu::{ArrayElement,array::ArrayContextExt,- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddSwapKernel},+ backends::common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Kernels, kernel::TensorAddSwapKernel,+ },};struct Input<T: ArrayElement + Float> {@@ -46,17 +49,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");let main_array = context.create_array_from(&[size], &input.main_buffer, "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(- skip_array.buffer().borrow_mut().deref_mut(),- main_array.buffer().borrow_mut().deref_mut(),- input.length,- encoder,- )- });- command_buffer.submit();- command_buffer.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ skip_array.buffer().borrow_mut().deref_mut(),+ main_array.buffer().borrow_mut().deref_mut(),+ input.length,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();(skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())}diff --git a/crates/uzu/tests/kernel/tensor_copy_test.rs b/crates/uzu/tests/kernel/tensor_copy_test.rsindex 4870584f9..e56ceef41 100644--- a/crates/uzu/tests/kernel/tensor_copy_test.rs+++ b/crates/uzu/tests/kernel/tensor_copy_test.rs@@ -8,7 +8,10 @@ use num_traits::Float;use uzu::{ArrayElement,array::ArrayContextExt,- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorCopyKernel},+ backends::common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Kernels, kernel::TensorCopyKernel,+ },};struct Input<T: ArrayElement + Float> {@@ -45,17 +48,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let src_array = context.create_array_from(&[size], &input.src, "");let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(- src_array.buffer().borrow().deref(),- dst_array.buffer().borrow_mut().deref_mut(),- input.length,- encoder,- )- });- command_buffer.submit();- command_buffer.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ src_array.buffer().borrow().deref(),+ dst_array.buffer().borrow_mut().deref_mut(),+ input.length,+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();dst_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/kernel/token_copy_test.rs b/crates/uzu/tests/kernel/token_copy_test.rsindex fa47b5e77..78b2c95b9 100644--- a/crates/uzu/tests/kernel/token_copy_test.rs+++ b/crates/uzu/tests/kernel/token_copy_test.rs@@ -4,7 +4,8 @@ use uzu::{ArrayElement,array::ArrayContextExt,backends::common::{- Backend, CommandBuffer, Context, Kernels,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Kernels,kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},},};@@ -18,12 +19,13 @@ fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {let src_array = context.create_array_from(&[1], &[src_value], "");let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)- });- command_buffer.submit();- command_buffer.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ src_array.buffer().borrow().deref(),+ dst_array.buffer().borrow_mut().deref_mut(),+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let output = dst_array.as_slice::<u64>().to_vec();assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());@@ -38,12 +40,13 @@ fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {let src_array = context.create_array_from(&[1], &[src_value], "");let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");- command_buffer.with_compute_encoder(|encoder| {- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)- });- command_buffer.submit();- command_buffer.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(+ src_array.buffer().borrow().deref(),+ dst_array.buffer().borrow_mut().deref_mut(),+ &mut command_buffer,+ );+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let output = dst_array.as_slice::<u32>().to_vec();assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex 4573536af..e73dc063b 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -1,12 +1,13 @@#![cfg(any(target_os = "macos", target_os = "ios"))]-use metal::{MTLCommandBuffer, MTLCommandQueue};-use objc2::Message;use uzu::{DataType,array::ArrayContextExt,backends::{- common::{Backend, Context, kernel::kv_cache_update::KVCacheUpdate},+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, kernel::kv_cache_update::KVCacheUpdate,+ },metal::Metal,},forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},@@ -185,18 +186,15 @@ fn run_scenario(},};- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();-- let root_command_buffer = command_buffer.clone();+ let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();layer.update_after_acceptance(&scenario.accepted_suffix_indices,scenario.suffix_start,- &mut root_command_buffer.retain(),+ &mut command_buffer,&kv_cache_update,);- command_buffer.commit();- root_command_buffer.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();layer.register_accepted_tokens(&scenario.accepted_token_positions);diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rsindex 90b93f9d6..da5c15980 100644--- a/crates/uzu/tests/kv_cache_update_test.rs+++ b/crates/uzu/tests/kv_cache_update_test.rs@@ -3,11 +3,11 @@use std::{cell::RefCell, rc::Rc};use bytemuck;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use ndarray::{Array, Array3, s};use uzu::backends::{common::{- Backend, Context,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,gpu_types::Swap,kernel::kv_cache_update::{KVCacheUpdate, KVLayerData, create_swaps_direct},},@@ -106,8 +106,7 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {value_shape: [num_heads, seq_len, head_dim],};- let mut command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut command_buffer) {Ok(_) => {},Err(e) => {@@ -116,8 +115,7 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {},}- command_buffer.commit();- command_buffer.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 389f08496..0e1723233 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -2,13 +2,14 @@use bytemuck;use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use ndarray::Array2;use uzu::{DataType,backends::{common::{- Backend, Context,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context,kernel::matmul::{MatmulArguments, MatmulKernel},},metal::{Metal, choose_dispatch_descriptor},@@ -45,8 +46,7 @@ fn run_metal_matmul(let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();let mut arguments = MatmulArguments {a: &a_buf,a_offset: 0,@@ -64,11 +64,8 @@ fn run_metal_matmul(};MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");- let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut enc);- enc.end_encoding();- encode_result.expect("encode");- cb.commit();- cb.wait_until_completed();+ kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut command_buffer).expect("encode");+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();unsafe {let ptr = d_buf.contents().as_ptr() as *const bf16;diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rsindex 6d1164f1f..e79359560 100644--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs@@ -1,14 +1,15 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};+use metal::MTLBuffer;use objc2::{rc::Retained, runtime::ProtocolObject};use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{common::{- Backend, Kernels,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels,kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},},metal::Metal,@@ -75,8 +76,7 @@ fn gen_topk_ids_from_logits(let mut topk_probs_buf = alloc_buffer::<bf16>(ctx, t * k);// Use fused router+topk kernel- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16).expect("router_topk kernel");router_topk.encode(@@ -90,11 +90,9 @@ fn gen_topk_ids_from_logits(e as u32,k as u32,true,- &mut router_topk_encoder,+ &mut command_buffer,);- router_topk_encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let ids_ptr = topk_ids_buf.contents().as_ptr() as *const i32;let ids = unsafe { std::slice::from_raw_parts(ids_ptr, t * k) };@@ -126,8 +124,8 @@ fn test_counts_offsets_fused_parity_random() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer =+ ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel.encode(&topk_ids_buf,@@ -137,12 +135,10 @@ fn test_counts_offsets_fused_parity_random() {t as u32,e as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Verify offsetslet offsets_ptr = offsets_buf.contents().as_ptr() as *const u32;@@ -178,8 +174,7 @@ fn test_counts_offsets_fused_edge_cases() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -188,11 +183,9 @@ fn test_counts_offsets_fused_edge_cases() {t as u32,e as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };@@ -211,8 +204,7 @@ fn test_counts_offsets_fused_edge_cases() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -221,11 +213,9 @@ fn test_counts_offsets_fused_edge_cases() {t as u32,e as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };assert!(offsets_gpu.iter().all(|&v| v == 0));diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex 4359b624a..a30ea799f 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -1,11 +1,12 @@use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};+use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{common::{- Backend, Kernels,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,@@ -346,12 +347,11 @@ fn run_moe_parity_test_internal(// Encode ALL kernels in one command buffereprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();// Router + TopK (fused kernel)let router_topk =<<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk");- let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");router_topk.encode(&x_buf,&router_w_buf,@@ -363,13 +363,11 @@ fn run_moe_parity_test_internal(e as u32,k as u32,true,- &mut router_topk_encoder,+ &mut command_buffer,);- router_topk_encoder.end_encoding();let fused_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");fused_kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -378,11 +376,9 @@ fn run_moe_parity_test_internal(t as u32,e as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx).expect("scatter bases kernel");scatter_bases_kernel.encode(@@ -393,11 +389,9 @@ fn run_moe_parity_test_internal(num_blocks as u32,num_tiles as u32,0u32,- &mut scatter_bases_encoder,+ &mut command_buffer,);- scatter_bases_encoder.end_encoding();- let mut scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");let scatter_map_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");@@ -415,13 +409,12 @@ fn run_moe_parity_test_internal(num_blocks as u32,num_tiles as u32,&mut tok2row_buf,- &mut scatter_encoder,+ &mut command_buffer,);- scatter_encoder.end_encoding();let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");gather.encode(- &mut cb,+ &mut command_buffer,DataType::BF16,MoeGatherArguments {x_buffer: &x_buf,@@ -469,11 +462,10 @@ fn run_moe_parity_test_internal(silu_alpha,data_type: DataType::BF16,};- experts.encode(&mut cb, args);+ experts.encode(&mut command_buffer, args);let finalize =<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");finalize.encode(&tok2row_buf,&topk_probs_buf,@@ -482,13 +474,11 @@ fn run_moe_parity_test_internal(t as u32,d_model as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();eprintln!("[E2E] GPU execution completed");// Read GPU outputdiff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex af5e63c1e..8fd419e07 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -1,13 +1,13 @@use std::time::Instant;use half::bf16;-use metal::{MTLCommandBuffer, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{common::{- Backend,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context,kernel::moe::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,@@ -106,9 +106,9 @@ fn run_decode_case(let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);for _ in 0..warmup {- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();experts_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -138,16 +138,15 @@ fn run_decode_case(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();experts_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -177,8 +176,7 @@ fn run_decode_case(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -270,7 +268,7 @@ fn run_two_pass_prefill_case(let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);for _ in 0..warmup {- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -299,15 +297,14 @@ fn run_two_pass_prefill_case(silu_alpha: 1.702,data_type: DataType::BF16,};- experts_kernel.encode(&mut cb, args);- cb.commit();- cb.wait_until_completed();+ experts_kernel.encode(&mut command_buffer, args);+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -336,9 +333,8 @@ fn run_two_pass_prefill_case(silu_alpha: 1.702,data_type: DataType::BF16,};- experts_kernel.encode(&mut cb, args);- cb.commit();- cb.wait_until_completed();+ experts_kernel.encode(&mut command_buffer, args);+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -413,9 +409,9 @@ fn run_fused_single_token_case(let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);for _ in 0..warmup {- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();fused_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -438,16 +434,15 @@ fn run_fused_single_token_case(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();fused_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -470,8 +465,7 @@ fn run_fused_single_token_case(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -593,9 +587,9 @@ fn run_indirect_decode_timed(let mut row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);for _ in 0..warmup {- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();experts_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -625,16 +619,15 @@ fn run_indirect_decode_timed(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();experts_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -664,8 +657,7 @@ fn run_indirect_decode_timed(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -712,9 +704,9 @@ fn run_fused_decode_timed(let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);for _ in 0..warmup {- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();fused_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -737,16 +729,15 @@ fn run_fused_decode_timed(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();fused_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -769,8 +760,7 @@ fn run_fused_decode_timed(data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex 15634b047..0afb09350 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -9,14 +9,17 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};+use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::moe::{- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,- MoeExpertsTwoPassPrefillBlock,+ common::{+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ kernel::moe::{+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,+ MoeExpertsTwoPassPrefillBlock,+ },},metal::Metal,},@@ -416,13 +419,13 @@ fn test_two_pass_decode_correctness() {// Execute 2-pass decode kernellet experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();const K_TILE: usize = 64;let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;experts_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -453,8 +456,7 @@ fn test_two_pass_decode_correctness() {},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Read GPU partial output and do CPU finalize (weighted sum)let y_partial_gpu =@@ -560,9 +562,9 @@ fn test_two_pass_decode_multi_token() {let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();experts_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -592,8 +594,7 @@ fn test_two_pass_decode_multi_token() {data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let y_partial_gpu =unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };@@ -662,7 +663,7 @@ fn test_two_pass_prefill_correctness() {let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -691,9 +692,8 @@ fn test_two_pass_prefill_correctness() {silu_alpha,data_type: DataType::BF16,};- experts_kernel.encode(&mut cb, args);- cb.commit();- cb.wait_until_completed();+ experts_kernel.encode(&mut command_buffer, args);+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let y_partial_gpu =unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };@@ -791,10 +791,10 @@ fn test_fused_single_token_decode() {// Run fused decode kernellet fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();fused_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -818,8 +818,7 @@ fn test_fused_single_token_decode() {},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Read GPU outputlet y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };@@ -915,9 +914,9 @@ fn test_fused_single_token_k4() {let mut y_buf = alloc_buffer::<bf16>(&ctx, d_model);let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();fused_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -940,8 +939,7 @@ fn test_fused_single_token_k4() {data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rsindex e292058b0..3677d3a5c 100644--- a/crates/uzu/tests/moe/moe_finalize_test.rs+++ b/crates/uzu/tests/moe/moe_finalize_test.rs@@ -1,12 +1,15 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};+use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::{Backend, Kernels, kernel::MoeFinalizeKernel},+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::MoeFinalizeKernel,+ },metal::Metal,},};@@ -90,8 +93,7 @@ fn test_finalize_correctness() {// Execute finalize kernellet finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();finalize.encode(&tok2row_buf,&probs_buf,@@ -100,11 +102,9 @@ fn test_finalize_correctness() {t as u32,d_model as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Comparelet y_gpu = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rsindex 7dfedae9b..f9cf5a01a 100644--- a/crates/uzu/tests/moe/moe_gather_test.rs+++ b/crates/uzu/tests/moe/moe_gather_test.rs@@ -1,12 +1,15 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};+use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::kernel::moe::{MoeGatherArguments, MoeGatherKernels},+ common::{+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ kernel::moe::{MoeGatherArguments, MoeGatherKernels},+ },metal::Metal,},};@@ -78,9 +81,9 @@ fn test_gather_correctness() {// Execute gather kernel using kernel structlet gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();gather.encode(- &mut cb,+ &mut command_buffer,DataType::BF16,MoeGatherArguments {x_buffer: &x_buf,@@ -92,8 +95,7 @@ fn test_gather_correctness() {d_model,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Comparelet x_gpu =diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex 7d2b61310..1e734200c 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -1,13 +1,13 @@use std::time::Instant;use half::bf16;-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{common::{- Backend, Kernels,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,@@ -121,8 +121,8 @@ fn test_moe_e2e_decode_perf() {// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer =+ ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();router_topk.encode(&x_buf,&router_w_buf,@@ -134,11 +134,9 @@ fn test_moe_e2e_decode_perf() {e as u32,k as u32,true,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});fused_perf.print();@@ -182,8 +180,8 @@ fn test_moe_e2e_prefill_perf() {// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer =+ ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();router_topk.encode(&x_buf,&router_w_buf,@@ -195,11 +193,9 @@ fn test_moe_e2e_prefill_perf() {e as u32,k as u32,true,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});fused_perf.print();@@ -311,8 +307,7 @@ fn test_moe_pipeline_breakdown_decode() {// Testing: Router + TopK + Counts+Offsets (FUSED)let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();router_topk_fused_kernel.encode(&x_buf,&router_w_buf,@@ -324,16 +319,13 @@ fn test_moe_pipeline_breakdown_decode() {e as u32,k as u32,true,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();counts_offsets_kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -342,16 +334,13 @@ fn test_moe_pipeline_breakdown_decode() {t as u32,e as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});let scatter_perf = time_kernel("Scatter", 2, 5, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();scatter_bases_kernel.encode(&partials_buf,&mut block_bases_buf,@@ -360,11 +349,8 @@ fn test_moe_pipeline_breakdown_decode() {num_blocks as u32,num_tiles as u32,0u32,- &mut scatter_bases_encoder,+ &mut command_buffer,);- scatter_bases_encoder.end_encoding();-- let mut scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");scatter_map_kernel.encode(&topk_ids_buf,&topk_probs_buf,@@ -379,18 +365,15 @@ fn test_moe_pipeline_breakdown_decode() {num_blocks as u32,num_tiles as u32,&mut tok2row_buf,- &mut scatter_map_encoder,+ &mut command_buffer,);- scatter_map_encoder.end_encoding();-- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});let gather_perf = time_kernel("Gather", 2, 5, || {- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();gather_kernel.encode(- &mut cb,+ &mut command_buffer,DataType::BF16,MoeGatherArguments {x_buffer: &x_buf,@@ -402,14 +385,13 @@ fn test_moe_pipeline_breakdown_decode() {d_model,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();experts_kernel.encode(- &mut cb,+ &mut command_buffer,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -439,13 +421,11 @@ fn test_moe_pipeline_breakdown_decode() {data_type: DataType::BF16,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});let finalize_perf = time_kernel("Finalize", 2, 5, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();finalize_kernel.encode(&tok2row_buf,&topk_probs_buf,@@ -454,11 +434,9 @@ fn test_moe_pipeline_breakdown_decode() {t as u32,d_model as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();});// Print resultsdiff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rsindex 9fc90a90b..cb869b65b 100644--- a/crates/uzu/tests/moe/moe_router_topk_test.rs+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs@@ -1,12 +1,15 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};+use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::{Backend, Kernels, kernel::MoeRouterTopKKernel},+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::MoeRouterTopKKernel,+ },metal::Metal,},};@@ -157,8 +160,7 @@ fn run_router_topk_once(// For BFloat16 kernel, probs buffer must be bf16, not f32let mut probs_buf = alloc_buffer::<bf16>(ctx, t * k);- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("Failed to create command encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel.encode(&input_buf,&weight_buf,@@ -170,11 +172,9 @@ fn run_router_topk_once(e as u32,k as u32,renorm,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let ids_ptr = ids_buf.contents().as_ptr() as *const i32;let probs_ptr = probs_buf.contents().as_ptr() as *const bf16;diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rsindex 3faf73fc3..a00356825 100644--- a/crates/uzu/tests/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/moe/moe_scatter_test.rs@@ -1,13 +1,14 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};+use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{common::{- Backend, Kernels,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,MoeScatterBucketsKernel,@@ -84,8 +85,7 @@ fn test_scatter_buckets_parity() {// Use fused router+topk kernellet router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router_topk");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();router_topk.encode(&input_buf,&weight_buf,@@ -97,11 +97,9 @@ fn test_scatter_buckets_parity() {e as u32,k as u32,true,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Read back CPU probs for reference comparelet probs_bf16 =@@ -116,8 +114,7 @@ fn test_scatter_buckets_parity() {let fused_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();fused_kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -126,11 +123,9 @@ fn test_scatter_buckets_parity() {t as u32,e as u32,k as u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Partials already created by fused kernel abovelet num_blocks = 1; // Fused kernel uses single block@@ -143,11 +138,9 @@ fn test_scatter_buckets_parity() {let scatter_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, DataType::BF16).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();let mut block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);- let mut scatter_bases_encoder =- cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");scatter_bases_kernel.encode(&partials_buf,&mut block_bases_buf,@@ -156,19 +149,16 @@ fn test_scatter_buckets_parity() {num_blocks as u32,num_tiles as u32,0u32,- &mut scatter_bases_encoder,+ &mut command_buffer,);- scatter_bases_encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let sumk = unsafe { std::slice::from_raw_parts(sumk_buf.contents().as_ptr() as *const u32, 1) }[0] as usize;let mut out_ids_buf = alloc_buffer::<i32>(&ctx, sumk);let mut out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();scatter_kernel.encode(&topk_ids_buf,&topk_probs_buf,@@ -182,12 +172,10 @@ fn test_scatter_buckets_parity() {k as u32,num_blocks as u32,num_tiles as u32,- &mut scatter_encoder,+ &mut command_buffer,);- scatter_encoder.end_encoding();- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let out_ids = unsafe { std::slice::from_raw_parts(out_ids_buf.contents().as_ptr() as *const i32, sumk) };let out_probs_h = unsafe { std::slice::from_raw_parts(out_probs_buf.contents().as_ptr() as *const bf16, sumk) };diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rsindex b0bd67301..9ab1ffc3e 100644--- a/crates/uzu/tests/moe/moe_tiles_test.rs+++ b/crates/uzu/tests/moe/moe_tiles_test.rs@@ -1,9 +1,12 @@#![cfg(any(target_os = "macos", target_os = "ios"))]-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};+use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::backends::{- common::kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},+ common::{+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},+ },metal::Metal,};@@ -35,17 +38,16 @@ fn test_tile_counts_correctness() {// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();tile_kernel.encode_counts(- &mut cb,+ &mut command_buffer,MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &mut tile_counts_buf,e,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Comparelet tile_counts_gpu =@@ -80,9 +82,9 @@ fn test_tile_scan_correctness() {// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();tile_kernel.encode_scan(- &mut cb,+ &mut command_buffer,MoeTileScanArguments {tile_counts_buffer: &tile_counts_buf,tile_offsets_buffer: &mut tile_offsets_buf,@@ -90,8 +92,7 @@ fn test_tile_scan_correctness() {e,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Comparelet tile_offsets_gpu =@@ -120,17 +121,16 @@ fn test_tile_edge_cases() {let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();tile_kernel.encode_counts(- &mut cb,+ &mut command_buffer,MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &mut tile_counts_buf,e,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let tile_counts_gpu =unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };@@ -151,17 +151,16 @@ fn test_tile_edge_cases() {let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();tile_kernel.encode_counts(- &mut cb,+ &mut command_buffer,MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &mut tile_counts_buf,e,},);- cb.commit();- cb.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let tile_counts_gpu =unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsindex 070956736..acedd70ea 100644--- a/crates/uzu/tests/quant_matmul_test.rs+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -2,13 +2,14 @@ use std::time::Instant;use bytemuck;use half::{bf16, f16};-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use objc2::{rc::Retained, runtime::ProtocolObject};use uzu::{DataType,backends::{common::{- Backend, Context,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context,kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,QuantizedMatmulType,@@ -510,12 +511,9 @@ fn execute_quantized_matmul(output_dim,quantization_type,};- let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&mut encoder, args).unwrap();- encoder.end_encoding();- cb_ref.commit();- cb_ref.wait_until_completed();+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();+ kernel.encode(&mut command_buffer, args).unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}}@@ -533,12 +531,9 @@ fn execute_quantized_matmul(output_dim,quantization_type,};- let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&mut encoder, args).unwrap();- encoder.end_encoding();- cb_ref.commit();- cb_ref.wait_until_completed();+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();+ kernel.encode(&mut command_buffer, args).unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();}let elapsed = start.elapsed();diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex 5650f6b69..dd2839838 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -1,13 +1,14 @@mod common;use bytemuck;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use rand::{RngExt, seq::SliceRandom};// for Vec::shuffleuse uzu::{DataType,backends::{common::{- Backend, CommandBuffer, Context, Kernels,+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,+ CommandBufferPending, Context, Kernels,kernel::{GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,sampling::{ArgmaxStrategy, SamplingKernel},@@ -107,13 +108,9 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {.new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();-- // Run sampling- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buffer,None,0,@@ -123,13 +120,10 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {SamplingMethod::Greedy,batch_size,vocab_size,- &mut compute_encoder,+ &mut command_buffer,).expect("Argmax sampling should succeed");- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();// Check resultslet result_ptr = output_buffer.contents().as_ptr() as *const u32;@@ -238,11 +232,10 @@ fn test_topp_sampling_from_prob_exact_match(.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let cb = cb_ref.to_owned();- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buf,Some(&seeds_buf),0,@@ -257,12 +250,10 @@ fn test_topp_sampling_from_prob_exact_match(},batch_size,vocab_size,- &mut compute_encoder,+ &mut command_buffer,).expect("encode");- compute_encoder.end_encoding();- cb_ref.commit();- cb_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let ptr = output_buf.contents().as_ptr() as *const u32;let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };@@ -361,12 +352,11 @@ fn test_topp_sampling_statistical_large() {.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let cb = cb_ref.to_owned();- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buf,Some(&seeds_buf),0,@@ -381,13 +371,10 @@ fn test_topp_sampling_statistical_large() {},BATCH,VOCAB,- &mut compute_encoder,+ &mut command_buffer,).expect("encode");- compute_encoder.end_encoding();-- cb_ref.commit();- cb_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let ptr = output_buf.contents().as_ptr() as *const u32;let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };@@ -463,12 +450,10 @@ fn perf_topp_128k_vocab() {.expect("Failed to create buffer");// ---- Launch once and time ----- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let cb = cb_ref.to_owned();- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buf,Some(&seeds_buf),0,@@ -483,19 +468,17 @@ fn perf_topp_128k_vocab() {},BATCH,VOCAB,- &mut compute_encoder,+ &mut command_buffer,).expect("encode");- compute_encoder.end_encoding();// Time both host-side and GPU executionlet host_timer = Instant::now();- cb_ref.commit();- cb_ref.wait_until_completed().unwrap();+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;// Get actual GPU execution time- let gpu_elapsed_ms = cb.gpu_execution_time_ms();+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();match gpu_elapsed_ms {Some(gpu_time) => {@@ -567,12 +550,10 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {.expect("Failed to create buffer");// ---- Launch once and time ----- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let cb = cb_ref.to_owned();- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buf,Some(&seeds_buf),0,@@ -582,19 +563,17 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {SamplingMethod::Greedy,BATCH,VOCAB,- &mut compute_encoder,+ &mut command_buffer,).expect("encode");- compute_encoder.end_encoding();// Time both host-side and GPU executionlet host_timer = Instant::now();- cb_ref.commit();- cb_ref.wait_until_completed().unwrap();+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;// Get actual GPU execution time- let gpu_elapsed_ms = cb.gpu_execution_time_ms();+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();match gpu_elapsed_ms {Some(gpu_time) => {@@ -686,13 +665,11 @@ fn test_categorical_sampling() {.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();- let mut compute_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buffer,Some(&seeds_buffer),0,@@ -707,13 +684,10 @@ fn test_categorical_sampling() {},batch_size,vocab_size,- &mut compute_encoder,+ &mut command_buffer,).expect("Categorical sampling should succeed");- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let result_ptr = output_buffer.contents().as_ptr() as *const u32;let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };@@ -828,13 +802,11 @@ fn test_categorical_sampling_statistical() {.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();- let mut compute_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buffer,Some(&seeds_buffer),0,@@ -849,13 +821,10 @@ fn test_categorical_sampling_statistical() {},BATCH,VOCAB,- &mut compute_encoder,+ &mut command_buffer,).expect("encode");- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let result_ptr = output_buffer.contents().as_ptr() as *const u32;let results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH) };@@ -935,12 +904,10 @@ fn perf_categorical_128k_vocab() {.expect("Failed to create buffer");// Launch and time- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let cb = cb_ref.to_owned();- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buf,Some(&seeds_buf),0,@@ -955,17 +922,15 @@ fn perf_categorical_128k_vocab() {},BATCH,VOCAB,- &mut compute_encoder,+ &mut command_buffer,).expect("encode");- compute_encoder.end_encoding();let host_timer = Instant::now();- cb_ref.commit();- cb_ref.wait_until_completed().unwrap();+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- let gpu_elapsed_ms = cb.gpu_execution_time_ms();+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();match gpu_elapsed_ms {Some(gpu_time) => {@@ -1023,22 +988,16 @@ fn test_temperature_gpu_cpu_match() {.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();-- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel.encode(Some(&logits_buffer),&mut processed_buffer,BATCH as u32,VOCAB as u32,TEMPERATURE,- &mut compute_encoder,+ &mut command_buffer,);- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };@@ -1096,15 +1055,9 @@ fn test_topk_gpu_cpu_match() {.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();-- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut compute_encoder);- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut command_buffer);+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let results_ptr = processed_buffer.contents().as_ptr() as *const f32;let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };@@ -1161,15 +1114,9 @@ fn test_topp_gpu_cpu_match() {.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();-- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut compute_encoder);- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut command_buffer);+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let results_ptr = processed_buffer.contents().as_ptr() as *const f32;let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };@@ -1231,15 +1178,9 @@ fn test_minp_gpu_cpu_match() {.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();-- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut compute_encoder);- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut command_buffer);+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let results_ptr = processed_buffer.contents().as_ptr() as *const f32;let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };@@ -1320,11 +1261,10 @@ fn test_minp_sampling_exact_match(.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let cb = cb_ref.to_owned();- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer =+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel- .encode_with_encoder(+ .encode(&mut logits_buf,Some(&seeds_buf),0,@@ -1339,12 +1279,10 @@ fn test_minp_sampling_exact_match(},batch_size,vocab_size,- &mut compute_encoder,+ &mut command_buffer,).expect("encode");- compute_encoder.end_encoding();- cb_ref.commit();- cb_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let ptr = output_buf.contents().as_ptr() as *const u32;let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };@@ -1418,22 +1356,16 @@ fn test_gumbel_gpu_cpu_match() {.new_buffer(BATCH * VOCAB * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");- let command_buffer = command_buffer_ref.to_owned();-- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();kernel.encode(Some(&logits_buffer),&seeds_buffer,&mut gumbel_logits_buffer,BATCH as u32,VOCAB as u32,- &mut compute_encoder,+ &mut command_buffer,);- compute_encoder.end_encoding();-- command_buffer_ref.commit();- command_buffer_ref.wait_until_completed().unwrap();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let result_ptr = gumbel_logits_buffer.contents().as_ptr() as *const f32;let all_results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex 470a714f8..70ca79769 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -3,16 +3,14 @@use std::mem::size_of;use bytemuck;-use metal::{- MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt,- MTLResourceOptions,-};+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use objc2::runtime::ProtocolObject;use uzu::{DataType,backends::{common::{- Backend, Context, Kernels,+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels,kernel::{Conv1dScanKernel,ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},@@ -248,12 +246,9 @@ fn run_prefill_kernel_mode(head_dim: fixture.head_dim,};- let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&mut encoder, args, mode);- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();+ kernel.encode(&mut command_buffer, args, mode);+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let y_vec = read_buffer(&y_buf, fixture.total_x);let state_vec = read_buffer(&state_buf, fixture.total_state);@@ -290,7 +285,8 @@ fn run_conv_scan_once(device.new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE).expect("Failed to create buffer");let b_buf =device.new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE).expect("Failed to create buffer");- let state_buf = device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");+ let mut state_buf =+ device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");let scratch_buf = if use_scratch && tap_count > 0 {Some(device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer"))} else {@@ -320,8 +316,7 @@ fn run_conv_scan_once(write_buffer(&padded_buf, &host);}- let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();let mut b_out = y_buf.clone();let mut c_out = y_buf.clone();let mut state_out = scratch_buf.as_ref().unwrap_or(&state_buf).clone();@@ -340,21 +335,17 @@ fn run_conv_scan_once(channels as u32,channels as u32,0u32,- &mut encoder,+ &mut command_buffer,);- encoder.end_encoding();if let Some(ref scratch) = scratch_buf {let bytes = channels * tap_count * size_of::<f32>();if bytes > 0 {- let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit encoder");- blit.copy_buffer_to_buffer(scratch, 0, &state_buf, 0, bytes);- blit.end_encoding();+ command_buffer.encode_copy(scratch, &mut state_buf, bytes);}}- command_buffer.commit();- command_buffer.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let y_vec = read_buffer(&y_buf, total_x);let state_vec = read_buffer(&state_buf, total_state);diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex af3377d2f..9567e0434 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -2,11 +2,14 @@use bytemuck;use half::bf16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};use uzu::{DataType,backends::{- common::{Backend, Context, Kernels, kernel::SSDUpdateKernel},+ common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context, Kernels, kernel::SSDUpdateKernel,+ },metal::Metal,},};@@ -239,9 +242,7 @@ fn ssd_update_with_z_bf16() {.expect("Failed to create buffer");let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16, false).unwrap();- let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let cb = cb_ref.to_owned();- let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();kernel.encode(&x_buf,&dt_buf,@@ -261,11 +262,9 @@ fn ssd_update_with_z_bf16() {bsz as u32,h as u32,dh as u32,- &mut enc,+ &mut command_buffer,);- enc.end_encoding();- cb_ref.commit();- cb_ref.wait_until_completed();+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();let y_ptr = y_buf.contents().as_ptr() as *const bf16;let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };