Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

new encoder (#236)

uuuvn5cfcf873d7d6
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rs
index ffb269a5d..6b0e8fd4a 100644
--- a/crates/uzu/build/common/traitgen.rs
+++ b/crates/uzu/build/common/traitgen.rs
@@ -19,7 +19,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
let params = kernel.parameters.iter().map(|p| {
let name = format_ident!("{}", p.name.as_ref());
let ty = match &p.ty {
- KernelParameterType::Type => quote! { DataType },
+ KernelParameterType::Type => quote! { crate::DataType },
KernelParameterType::Value(ty) => {
let ty: Type = syn::parse_str(ty.as_ref()).unwrap();
quote! { #ty }
@@ -42,10 +42,10 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
Some(quote! { #buffer_lifetime }),
match access {
KernelBufferAccess::Read => {
- quote! { impl BufferArg<#buffer_lifetime, <Self::Backend as Backend>::NativeBuffer> }
+ quote! { impl BufferArg<#buffer_lifetime, <Self::Backend as crate::backends::common::Backend>::Buffer> }
},
KernelBufferAccess::ReadWrite => {
- quote! { impl BufferArgMut<#buffer_lifetime, <Self::Backend as Backend>::NativeBuffer> }
+ quote! { impl BufferArgMut<#buffer_lifetime, <Self::Backend as crate::backends::common::Backend>::Buffer> }
},
},
)
@@ -72,12 +72,11 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
let kernel_trait = quote! {
pub trait #trait_name: Sized {
- type Backend: Backend<Kernels: Kernels<#trait_name = Self>>;
+ type Backend: crate::backends::common::Backend<Kernels: Kernels<#trait_name = Self>>;
- fn new(context: &<Self::Backend as Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as Backend>::Error>;
+ fn new(context: &<Self::Backend as crate::backends::common::Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as crate::backends::common::Backend>::Error>;
- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder);
- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder, predicate: Option<impl BufferArg<'predicate, <Self::Backend as Backend>::NativeBuffer>>);
+ fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#args ,)* command_buffer: &'command_buffer mut <<Self::Backend as crate::backends::common::Backend>::CommandBuffer as crate::backends::common::CommandBuffer>::Encoding);
}
};
@@ -114,36 +113,33 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>
}
let kernel_traits = quote! {
- use crate::backends::common::{Backend, NativeBuffer};
- use crate::DataType;
-
- pub trait BufferArg<'a, B: NativeBuffer> {
+ pub trait BufferArg<'a, B: crate::backends::common::Buffer> {
fn into_parts(self) -> (&'a B, usize);
}
- impl<'a, B: NativeBuffer> BufferArg<'a, B> for &'a B {
+ impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for &'a B {
fn into_parts(self) -> (&'a B, usize) {
(self, 0)
}
}
- impl<'a, B: NativeBuffer> BufferArg<'a, B> for (&'a B, usize) {
+ impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for (&'a B, usize) {
fn into_parts(self) -> (&'a B, usize) {
self
}
}
- pub trait BufferArgMut<'a, B: NativeBuffer> {
+ pub trait BufferArgMut<'a, B: crate::backends::common::Buffer> {
fn into_parts(self) -> (&'a mut B, usize);
}
- impl<'a, B: NativeBuffer> BufferArgMut<'a, B> for &'a mut B {
+ impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for &'a mut B {
fn into_parts(self) -> (&'a mut B, usize) {
(self, 0)
}
}
- impl<'a, B: NativeBuffer> BufferArgMut<'a, B> for (&'a mut B, usize) {
+ impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for (&'a mut B, usize) {
fn into_parts(self) -> (&'a mut B, usize) {
self
}
@@ -152,7 +148,7 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>
#(#kernel_traits)*
pub trait Kernels: Sized {
- type Backend: Backend<Kernels = Self>;
+ type Backend: crate::backends::common::Backend<Kernels = Self>;
#(#kernel_types)*
}
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rs
index 971d14281..da8c99615 100644
--- a/crates/uzu/build/cpu/compiler.rs
+++ b/crates/uzu/build/cpu/compiler.rs
@@ -329,7 +329,7 @@ impl CpuCompiler {
})
.collect::<anyhow::Result<_>>()?;
- let (encode_generics, encode_args_defs, encode_args_passthrough): (Vec<_>, Vec<_>, Vec<_>) = kernel_arguments
+ let (encode_generics, encode_args_defs): (Vec<_>, Vec<_>) = kernel_arguments
.iter()
.map(|argument| {
let argument_ident: Ident = syn::parse_str(argument.name.as_ref()).context("cannot parse ident")?;
@@ -359,7 +359,7 @@ impl CpuCompiler {
ty = quote! { Option<#ty> };
}
- Ok((generic, quote! { #argument_ident: #ty }, quote! { #argument_ident }))
+ Ok((generic, quote! { #argument_ident: #ty }))
})
.collect::<anyhow::Result<_>>()?;
@@ -439,12 +439,7 @@ impl CpuCompiler {
});
quote! {
- encoder.push_command(move || {
- if let Some(predicate) = predicate && unsafe { *predicate } == 0u32 {
- return;
- }
- #monomorphized_function(#function_call_args_joined)
- });
+ encoder.push_command(move || #monomorphized_function(#function_call_args_joined));
}
};
@@ -536,16 +531,7 @@ impl CpuCompiler {
}
fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer) {
- self.encode_if(#(#encode_args_passthrough ,)* encoder, None::<&Box<[u8]>>)
- }
-
- fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {
#(#argument_copies)*
- let predicate = predicate.map(|__dsl_buffer_impl| unsafe {
- let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();
-
- __dsl_buffer.as_ptr().byte_add(__dsl_offset) as *const u32
- });
#encode_body
}
}
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index 0db3014f3..309c6bb7a 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -99,13 +99,12 @@ pub fn bindgen(
let mut arg_count: usize = 0;
let mut indirect_flag = false;
- let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_args_sets, encode_args_names): (
+ let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_args_sets): (
Vec<Option<TokenStream>>,
Vec<Option<TokenStream>>,
Vec<Option<TokenStream>>,
Vec<TokenStream>,
Vec<TokenStream>,
- Vec<TokenStream>,
) = kernel
.arguments
.iter()
@@ -176,13 +175,12 @@ pub fn bindgen(
generic,
quote! { #arg_name: #ty },
set,
- quote! { #arg_name },
))
}
MetalArgumentType::Groups(MetalGroupsType::Indirect) if !indirect_flag => {
indirect_flag = true;
- Some((None, None, Some(quote! { '__dsl_indirect_dispatch_buffer }), quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> }, quote! {}, quote! { __dsl_indirect_dispatch_buffer }))
+ Some((None, None, Some(quote! { '__dsl_indirect_dispatch_buffer }), quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> }, quote! {}))
}
_ => None,
}
@@ -323,29 +321,13 @@ pub fn bindgen(
Ok(Self { pipeline #(, #conditional_buffer_sets)* })
}
- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {
+ fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {
+ let compute_encoder = command_buffer.ensure_compute();
#empty_dispatch_guards
compute_encoder.set_compute_pipeline_state(&self.pipeline);
#(#encode_args_sets)*
#dispatch
}
- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {
- #empty_dispatch_guards
-
- if let Some(predicate) = predicate {
- let (__dsl_buffer, __dsl_offset) = predicate.into_parts();
- compute_encoder.condition(
- __dsl_buffer,
- __dsl_offset,
- |compute_encoder| {
- self.encode(#(#encode_args_names, )* compute_encoder);
- },
- None::<fn(&mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>)>,
- );
- } else {
- self.encode(#(#encode_args_names, )* compute_encoder);
- }
- }
}
};
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
index 1120c6848..ef53773b3 100644
--- a/crates/uzu/build/metal/compiler.rs
+++ b/crates/uzu/build/metal/compiler.rs
@@ -288,9 +288,10 @@ impl MetalCompiler {
use crate::backends::metal::{
context::MetalContext,
+ command_buffer::MetalCommandBufferEncoding,
error::MetalError,
metal_extensions::{
- ComputeEncoderConditional, ComputeEncoderSetValue, FunctionConstantValuesSetValue, MetalDataTypeExt,
+ ComputeEncoderSetValue, FunctionConstantValuesSetValue, MetalDataTypeExt,
},
};
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rs
index 20b7f16e1..2c3c3fe9b 100644
--- a/crates/uzu/src/array.rs
+++ b/crates/uzu/src/array.rs
@@ -4,12 +4,12 @@ use ndarray::{ArrayView, Dimension, IxDyn};
use crate::{
ArrayElement, DataType,
- backends::common::{Backend, Context, NativeBuffer},
+ backends::common::{Backend, Buffer, Context},
};
#[derive(Debug)]
pub struct Array<B: Backend> {
- buffer: Rc<RefCell<B::NativeBuffer>>,
+ buffer: Rc<RefCell<B::Buffer>>,
offset: usize,
shape: Box<[usize]>,
data_type: DataType,
@@ -18,7 +18,7 @@ pub struct Array<B: Backend> {
impl<B: Backend> Array<B> {
// Constructors
pub unsafe fn from_parts(
- buffer: Rc<RefCell<B::NativeBuffer>>,
+ buffer: Rc<RefCell<B::Buffer>>,
offset: usize,
shape: &[usize],
data_type: DataType,
@@ -49,7 +49,7 @@ impl<B: Backend> Array<B> {
}
// Getters
- pub fn buffer(&self) -> Rc<RefCell<B::NativeBuffer>> {
+ pub fn buffer(&self) -> Rc<RefCell<B::Buffer>> {
self.buffer.clone()
}
diff --git a/crates/uzu/src/backends/common/allocator/allocator.rs b/crates/uzu/src/backends/common/allocator/allocator.rs
deleted file mode 100644
index 66bf849b1..000000000
--- a/crates/uzu/src/backends/common/allocator/allocator.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-use std::{
- cell::{Cell, RefCell},
- rc::Weak,
-};
-
-use super::{
- AllocError,
- scratch_pool::{ScratchPool, align_size},
-};
-use crate::backends::common::{Backend, Buffer, BufferLifetime, Context, NativeBuffer};
-
-pub struct Allocator<B: Backend> {
- context: Weak<B::Context>,
- scratch_pool: RefCell<ScratchPool<B::NativeBuffer>>,
- active_memory: Cell<usize>,
- peak_memory: Cell<usize>,
- gc_limit: Cell<usize>,
- max_pool_size: Cell<usize>,
-}
-
-impl<B: Backend> Allocator<B> {
- pub fn new(context: Weak<B::Context>) -> Self {
- Self {
- context,
- scratch_pool: RefCell::new(ScratchPool::new()),
- active_memory: Cell::new(0),
- peak_memory: Cell::new(0),
- gc_limit: Cell::new(usize::MAX),
- max_pool_size: Cell::new(usize::MAX),
- }
- }
-
- fn create_native_buffer(
- &self,
- size: usize,
- ) -> Result<B::NativeBuffer, AllocError> {
- let context = self.context.upgrade().ok_or_else(|| AllocError::AllocationFailed {
- size,
- reason: "context has been dropped".to_string(),
- })?;
- context.create_buffer(size).map_err(|e| AllocError::AllocationFailed {
- size,
- reason: e.to_string(),
- })
- }
-
- fn wrap_buffer(
- &self,
- native: B::NativeBuffer,
- is_scratch: bool,
- ) -> Buffer<B> {
- Buffer::new(native, self.context.clone(), is_scratch)
- }
-
- pub fn set_gc_limit(
- &self,
- limit: usize,
- ) {
- self.gc_limit.set(limit);
- }
-
- pub fn set_max_pool_size(
- &self,
- limit: usize,
- ) {
- self.max_pool_size.set(limit);
- }
-
- pub fn gc_limit(&self) -> usize {
- self.gc_limit.get()
- }
-
- pub fn max_pool_size(&self) -> usize {
- self.max_pool_size.get()
- }
-
- #[inline]
- fn track_allocation(
- &self,
- size: usize,
- ) {
- let new_active = self.active_memory.get() + size;
- self.active_memory.set(new_active);
- if new_active > self.peak_memory.get() {
- self.peak_memory.set(new_active);
- }
- }
-
- #[inline]
- fn track_deallocation(
- &self,
- size: usize,
- ) {
- self.active_memory.set(self.active_memory.get().saturating_sub(size));
- }
-
- fn alloc_scratch_internal(
- &self,
- aligned_size: usize,
- ) -> Result<B::NativeBuffer, AllocError> {
- let mut pool = self.scratch_pool.borrow_mut();
-
- if let Some((buffer, actual_size)) = pool.find_buffer(aligned_size) {
- self.track_allocation(actual_size);
- return Ok(buffer);
- }
-
- let gc_limit = self.gc_limit.get();
- let active = self.active_memory.get();
- let cache = pool.available_size();
- let mem_required = active + cache + aligned_size;
-
- if mem_required >= gc_limit {
- let to_free = mem_required.saturating_sub(gc_limit);
- pool.release_cached_buffers(to_free);
- }
-
- pool.total_allocated += aligned_size;
- drop(pool);
-
- let buffer = self.create_native_buffer(aligned_size)?;
- self.track_allocation(aligned_size);
-
- Ok(buffer)
- }
-
- pub fn alloc(
- &self,
- lifetime: BufferLifetime,
- size: usize,
- ) -> Result<Buffer<B>, AllocError> {
- let aligned_size = align_size(size);
-
- let (native, is_scratch) = match lifetime {
- BufferLifetime::Permanent => {
- let buffer = self.create_native_buffer(aligned_size)?;
- self.track_allocation(aligned_size);
- (buffer, false)
- },
- BufferLifetime::Scratch => (self.alloc_scratch_internal(aligned_size)?, true),
- };
-
- Ok(self.wrap_buffer(native, is_scratch))
- }
-
- pub fn handle_buffer_drop(
- &self,
- buffer: B::NativeBuffer,
- is_scratch: bool,
- ) {
- let size = buffer.length();
- self.track_deallocation(size);
-
- if is_scratch {
- let mut pool = self.scratch_pool.borrow_mut();
- pool.return_buffer(buffer, size);
-
- let max_pool_size = self.max_pool_size.get();
- let current_size = pool.available_size();
- if current_size > max_pool_size {
- let to_free = current_size - max_pool_size;
- pool.release_cached_buffers(to_free);
- }
- }
- }
-
- pub fn active_memory(&self) -> usize {
- self.active_memory.get()
- }
-
- pub fn peak_memory(&self) -> usize {
- self.peak_memory.get()
- }
-
- pub fn reset_peak_memory(&self) {
- self.peak_memory.set(self.active_memory.get());
- }
-
- pub fn cache_memory(&self) -> usize {
- self.scratch_pool.borrow().available_size()
- }
-
- pub fn clear_cache(&self) {
- self.scratch_pool.borrow_mut().clear();
- }
-}
diff --git a/crates/uzu/src/backends/common/allocator/error.rs b/crates/uzu/src/backends/common/allocator/error.rs
deleted file mode 100644
index bb109b172..000000000
--- a/crates/uzu/src/backends/common/allocator/error.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use std::fmt;
-
-#[derive(Debug, Clone)]
-pub enum AllocError {
- OutOfMemory {
- requested: usize,
- available: usize,
- },
- AllocationFailed {
- size: usize,
- reason: String,
- },
-}
-
-impl fmt::Display for AllocError {
- fn fmt(
- &self,
- f: &mut fmt::Formatter<'_>,
- ) -> fmt::Result {
- match self {
- AllocError::OutOfMemory {
- requested,
- available,
- } => {
- write!(f, "Out of memory: requested {} bytes, {} available", requested, available)
- },
- AllocError::AllocationFailed {
- size,
- reason,
- } => {
- write!(f, "Failed to allocate {} bytes: {}", size, reason)
- },
- }
- }
-}
-
-impl std::error::Error for AllocError {}
diff --git a/crates/uzu/src/backends/common/allocator/mod.rs b/crates/uzu/src/backends/common/allocator/mod.rs
deleted file mode 100644
index a7ffa15d5..000000000
--- a/crates/uzu/src/backends/common/allocator/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-mod allocator;
-mod error;
-mod scratch_pool;
-
-pub use allocator::Allocator;
-pub use error::AllocError;
diff --git a/crates/uzu/src/backends/common/allocator/scratch_pool.rs b/crates/uzu/src/backends/common/allocator/scratch_pool.rs
deleted file mode 100644
index 963deb3d2..000000000
--- a/crates/uzu/src/backends/common/allocator/scratch_pool.rs
+++ /dev/null
@@ -1,255 +0,0 @@
-use std::collections::BTreeMap;
-
-pub(super) const PAGE_SIZE: usize = 16384;
-
-pub(super) fn align_size(size: usize) -> usize {
- if size > PAGE_SIZE {
- (size + PAGE_SIZE - 1) & !(PAGE_SIZE - 1)
- } else {
- size
- }
-}
-
-struct BufferNode<B> {
- buffer: B,
- size: usize,
- bucket_idx: usize,
- prev: Option<usize>,
- next: Option<usize>,
-}
-
-pub(super) struct ScratchPool<B> {
- nodes: Vec<Option<BufferNode<B>>>,
- free_slots: Vec<usize>,
- size_index: BTreeMap<usize, Vec<usize>>,
- lru_head: Option<usize>,
- lru_tail: Option<usize>,
- pool_size: usize,
- pub(super) total_allocated: usize,
-}
-
-impl<B> ScratchPool<B> {
- pub(super) fn new() -> Self {
- Self {
- nodes: Vec::new(),
- free_slots: Vec::new(),
- size_index: BTreeMap::new(),
- lru_head: None,
- lru_tail: None,
- pool_size: 0,
- total_allocated: 0,
- }
- }
-
- #[inline]
- fn allocate_slot(&mut self) -> usize {
- self.free_slots.pop().unwrap_or_else(|| {
- let slot = self.nodes.len();
- self.nodes.push(None);
- slot
- })
- }
-
- #[inline]
- fn add_to_lru_head(
- &mut self,
- slot: usize,
- ) {
- if let Some(old_head) = self.lru_head {
- if let Some(node) = self.nodes[old_head].as_mut() {
- node.prev = Some(slot);
- }
- if let Some(node) = self.nodes[slot].as_mut() {
- node.next = Some(old_head);
- }
- }
-
- self.lru_head = Some(slot);
-
- if self.lru_tail.is_none() {
- self.lru_tail = Some(slot);
- }
- }
-
- #[inline]
- fn remove_from_lru(
- &mut self,
- slot: usize,
- ) {
- let (prev, next) = match self.nodes[slot].as_ref() {
- Some(node) => (node.prev, node.next),
- None => return,
- };
-
- match (prev, next) {
- (Some(p), Some(n)) => {
- if let Some(prev_node) = self.nodes[p].as_mut() {
- prev_node.next = Some(n);
- }
- if let Some(next_node) = self.nodes[n].as_mut() {
- next_node.prev = Some(p);
- }
- },
- (Some(p), None) => {
- if let Some(prev_node) = self.nodes[p].as_mut() {
- prev_node.next = None;
- }
- self.lru_tail = Some(p);
- },
- (None, Some(n)) => {
- if let Some(next_node) = self.nodes[n].as_mut() {
- next_node.prev = None;
- }
- self.lru_head = Some(n);
- },
- (None, None) => {
- self.lru_head = None;
- self.lru_tail = None;
- },
- }
-
- if let Some(node) = self.nodes[slot].as_mut() {
- node.prev = None;
- node.next = None;
- }
- }
-
- #[inline]
- fn remove_slot_from_bucket(
- &mut self,
- size: usize,
- _slot: usize,
- bucket_idx: usize,
- ) {
- if let Some(slots) = self.size_index.get_mut(&size) {
- let last_idx = slots.len() - 1;
- if bucket_idx != last_idx {
- let swapped_slot = slots[last_idx];
- slots[bucket_idx] = swapped_slot;
- if let Some(swapped_node) = self.nodes[swapped_slot].as_mut() {
- swapped_node.bucket_idx = bucket_idx;
- }
- }
- slots.pop();
- if slots.is_empty() {
- self.size_index.remove(&size);
- }
- }
- }
-
- #[inline]
- pub(super) fn find_buffer(
- &mut self,
- size: usize,
- ) -> Option<(B, usize)> {
- let max_size = (2 * size).min(size + 2 * PAGE_SIZE);
-
- let mut found_key = None;
- let mut found_slot = None;
- let mut found_bucket_idx = None;
-
- for (&key, slots) in self.size_index.range(size..=max_size) {
- if let Some(&slot) = slots.last() {
- found_key = Some(key);
- found_slot = Some(slot);
- found_bucket_idx = Some(slots.len() - 1);
- break;
- }
- }
-
- let (key, slot, bucket_idx) = match (found_key, found_slot, found_bucket_idx) {
- (Some(k), Some(s), Some(i)) => (k, s, i),
- _ => return None,
- };
-
- self.remove_slot_from_bucket(key, slot, bucket_idx);
- self.remove_from_lru(slot);
-
- let node = self.nodes[slot].take()?;
- self.free_slots.push(slot);
- self.pool_size = self.pool_size.saturating_sub(key);
-
- Some((node.buffer, key))
- }
-
- #[inline]
- pub(super) fn return_buffer(
- &mut self,
- buffer: B,
- size: usize,
- ) {
- let slot = self.allocate_slot();
- let bucket_idx = self.size_index.get(&size).map_or(0, |v| v.len());
-
- self.nodes[slot] = Some(BufferNode {
- buffer,
- size,
- bucket_idx,
- prev: None,
- next: None,
- });
-
- self.add_to_lru_head(slot);
- self.size_index.entry(size).or_default().push(slot);
- self.pool_size += size;
- }
-
- pub(super) fn release_cached_buffers(
- &mut self,
- min_bytes_to_free: usize,
- ) -> usize {
- if min_bytes_to_free == 0 {
- return 0;
- }
-
- if min_bytes_to_free >= (self.pool_size * 9) / 10 {
- return self.clear();
- }
-
- let mut total_freed = 0usize;
- let mut buffers_released = 0usize;
-
- while total_freed < min_bytes_to_free {
- let tail_slot = match self.lru_tail {
- Some(s) => s,
- None => break,
- };
-
- let (size, bucket_idx) = match self.nodes[tail_slot].as_ref() {
- Some(n) => (n.size, n.bucket_idx),
- None => break,
- };
-
- self.remove_slot_from_bucket(size, tail_slot, bucket_idx);
- self.remove_from_lru(tail_slot);
- self.nodes[tail_slot] = None;
- self.free_slots.push(tail_slot);
-
- total_freed += size;
- buffers_released += 1;
- self.pool_size = self.pool_size.saturating_sub(size);
- self.total_allocated = self.total_allocated.saturating_sub(size);
- }
-
- buffers_released
- }
-
- #[inline]
- pub(super) fn available_size(&self) -> usize {
- self.pool_size
- }
-
- pub(super) fn clear(&mut self) -> usize {
- let count = self.nodes.iter().filter(|n| n.is_some()).count();
-
- self.nodes.clear();
- self.free_slots.clear();
- self.size_index.clear();
- self.lru_head = None;
- self.lru_tail = None;
- self.total_allocated = self.total_allocated.saturating_sub(self.pool_size);
- self.pool_size = 0;
-
- count
- }
-}
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rs
index 9d811eb5c..e7bba72f3 100644
--- a/crates/uzu/src/backends/common/backend.rs
+++ b/crates/uzu/src/backends/common/backend.rs
@@ -1,13 +1,11 @@
use std::{error::Error, fmt::Debug};
-use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer, kernel::matmul::MatmulKernels};
+use super::{Buffer, CommandBuffer, Context, Event, Kernels, kernel::matmul::MatmulKernels};
pub trait Backend: Debug + Clone + 'static {
type Context: Context<Backend = Self>;
- type NativeBuffer: NativeBuffer<Backend = Self>;
type CommandBuffer: CommandBuffer<Backend = Self>;
- type ComputeEncoder;
- type CopyEncoder: CopyEncoder<Backend = Self>;
+ type Buffer: Buffer<Backend = Self>;
type Event: Event<Backend = Self>;
type Kernels: Kernels<Backend = Self> + MatmulKernels;
type Error: Error + Debug;
diff --git a/crates/uzu/src/backends/common/buffer.rs b/crates/uzu/src/backends/common/buffer.rs
index 8630de880..832ce2cc9 100644
--- a/crates/uzu/src/backends/common/buffer.rs
+++ b/crates/uzu/src/backends/common/buffer.rs
@@ -1,38 +1,18 @@
-use std::{mem::ManuallyDrop, rc::Weak};
+use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};
-use super::{Backend, Context};
+use super::Backend;
-pub struct Buffer<B: Backend> {
- inner: ManuallyDrop<B::NativeBuffer>,
- context: Weak<B::Context>,
- is_scratch: bool,
-}
+pub trait Buffer: Debug {
+ type Backend: Backend<Buffer = Self>;
-impl<B: Backend> Buffer<B> {
- pub fn new(
- inner: B::NativeBuffer,
- context: Weak<B::Context>,
- is_scratch: bool,
- ) -> Self {
- Self {
- inner: ManuallyDrop::new(inner),
- context,
- is_scratch,
- }
- }
+ fn set_label(
+ &mut self,
+ label: Option<&str>,
+ );
- pub fn inner(&self) -> &B::NativeBuffer {
- &self.inner
- }
-}
+ fn cpu_ptr(&self) -> NonNull<c_void>;
+
+ fn length(&self) -> usize;
-impl<B: Backend> Drop for Buffer<B> {
- fn drop(&mut self) {
- // Safety: drop is only called once, inner is valid
- let inner = unsafe { ManuallyDrop::take(&mut self.inner) };
- if let Some(ctx) = self.context.upgrade() {
- ctx.allocator().handle_buffer_drop(inner, self.is_scratch);
- }
- // If context is gone, inner drops naturally here
- }
+ fn id(&self) -> usize;
}
diff --git a/crates/uzu/src/backends/common/buffer_lifetime.rs b/crates/uzu/src/backends/common/buffer_lifetime.rs
deleted file mode 100644
index 65cf6b5a7..000000000
--- a/crates/uzu/src/backends/common/buffer_lifetime.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum BufferLifetime {
- Permanent,
- Scratch,
-}
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index a23e00622..a9bdb96cc 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -1,38 +1,84 @@
+use std::ops::Range;
+
use super::Backend;
pub trait CommandBuffer {
- type Backend: Backend;
+ type Backend: Backend<CommandBuffer = Self>;
+
+ type Initial: CommandBufferInitial<CommandBuffer = Self>;
+ type Encoding: CommandBufferEncoding<CommandBuffer = Self>;
+ type Executable: CommandBufferExecutable<CommandBuffer = Self>;
+ type Pending: CommandBufferPending<CommandBuffer = Self>;
+ type Completed: CommandBufferCompleted<CommandBuffer = Self>;
+}
+
+pub trait CommandBufferInitial {
+ type CommandBuffer: CommandBuffer<Initial = Self>;
- fn with_compute_encoder<T>(
+ fn start_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Encoding;
+}
+
+pub trait CommandBufferEncoding {
+ type CommandBuffer: CommandBuffer<Encoding = Self>;
+
+ fn encode_copy(
&mut self,
- callback: impl FnOnce(&mut <Self::Backend as Backend>::ComputeEncoder) -> T,
- ) -> T;
+ src: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,
+ dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,
+ size: usize,
+ );
- fn with_copy_encoder<T>(
+ fn encode_fill(
&mut self,
- callback: impl FnOnce(&mut <Self::Backend as Backend>::CopyEncoder) -> T,
- ) -> T;
+ dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,
+ range: Range<usize>,
+ value: u8,
+ );
fn encode_wait_for_event(
&mut self,
- event: &<Self::Backend as Backend>::Event,
+ event: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Event,
value: u64,
);
fn encode_signal_event(
&mut self,
- event: &<Self::Backend as Backend>::Event,
+ event: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Event,
value: u64,
);
fn add_completion_handler(
&mut self,
- handler: impl FnOnce(Result<(), <Self::Backend as Backend>::Error>) + 'static,
+ handler: impl FnOnce(
+ Result<
+ &<Self::CommandBuffer as CommandBuffer>::Completed,
+ <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
+ >,
+ ) + 'static,
);
- fn submit(&mut self);
+ fn end_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Executable;
+}
+
+pub trait CommandBufferExecutable {
+ type CommandBuffer: CommandBuffer<Executable = Self>;
+
+ fn submit(self) -> <Self::CommandBuffer as CommandBuffer>::Pending;
+}
+
+pub trait CommandBufferPending {
+ type CommandBuffer: CommandBuffer<Pending = Self>;
+
+ fn wait_until_completed(
+ self
+ ) -> Result<
+ <Self::CommandBuffer as CommandBuffer>::Completed,
+ <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Error,
+ >;
+}
- fn wait_until_completed(&self) -> Result<(), <Self::Backend as Backend>::Error>;
+pub trait CommandBufferCompleted {
+ type CommandBuffer: CommandBuffer<Completed = Self>;
fn gpu_execution_time_ms(&self) -> Option<f64>;
}
diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rs
index ca5a27cff..d4fb3fd05 100644
--- a/crates/uzu/src/backends/common/context.rs
+++ b/crates/uzu/src/backends/common/context.rs
@@ -1,6 +1,7 @@
use std::{path::Path, rc::Rc};
-use super::{Allocator, Backend};
+use super::Backend;
+use crate::backends::common::CommandBuffer;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeviceClass {
@@ -26,16 +27,14 @@ pub trait Context: Sized {
fn debug_active(&self) -> bool;
- fn allocator(&self) -> &Allocator<Self::Backend>;
+ fn create_command_buffer(
+ &self
+ ) -> Result<<<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Initial, <Self::Backend as Backend>::Error>;
fn create_buffer(
&self,
size: usize,
- ) -> Result<<Self::Backend as Backend>::NativeBuffer, <Self::Backend as Backend>::Error>;
-
- fn create_command_buffer(
- &self
- ) -> Result<<Self::Backend as Backend>::CommandBuffer, <Self::Backend as Backend>::Error>;
+ ) -> Result<<Self::Backend as Backend>::Buffer, <Self::Backend as Backend>::Error>;
fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;
diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rs
deleted file mode 100644
index 93c319ae2..000000000
--- a/crates/uzu/src/backends/common/copy_encoder.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-use std::ops::Range;
-
-use super::Backend;
-
-pub trait CopyEncoder {
- type Backend: Backend;
-
- fn encode_copy(
- &mut self,
- src: &<Self::Backend as Backend>::NativeBuffer,
- dst: &mut <Self::Backend as Backend>::NativeBuffer,
- size: usize,
- );
-
- fn encode_fill(
- &mut self,
- dst: &mut <Self::Backend as Backend>::NativeBuffer,
- range: Range<usize>,
- value: u8,
- );
-}
diff --git a/crates/uzu/src/backends/common/event.rs b/crates/uzu/src/backends/common/event.rs
index 57eeccf77..776b143cb 100644
--- a/crates/uzu/src/backends/common/event.rs
+++ b/crates/uzu/src/backends/common/event.rs
@@ -1,5 +1,5 @@
use super::Backend;
pub trait Event {
- type Backend: Backend;
+ type Backend: Backend<Event = Self>;
}
diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rs
index 8d8650504..c0522bdd7 100644
--- a/crates/uzu/src/backends/common/kernel/attention.rs
+++ b/crates/uzu/src/backends/common/kernel/attention.rs
@@ -6,7 +6,7 @@ use std::{
use crate::{
DataType,
backends::common::{
- Backend, Kernels,
+ Backend, CommandBuffer, Kernels,
gpu_types::{AttnMaskParams, AttnParams},
kernel::AttentionGemmKernel,
},
@@ -15,12 +15,12 @@ use crate::{
const BQ: usize = 32;
pub struct AttentionGemmArguments<'a, B: Backend> {
- pub queries_buffer: &'a B::NativeBuffer, // buffer(0)
- pub keys_buffer: &'a B::NativeBuffer, // buffer(1)
- pub values_buffer: &'a B::NativeBuffer, // buffer(2)
- pub output_buffer: &'a mut B::NativeBuffer, // buffer(3)
- pub mask_buffer: Option<&'a B::NativeBuffer>, // buffer(6)
- pub sinks_buffer: Option<&'a B::NativeBuffer>, // buffer(7)
+ pub queries_buffer: &'a B::Buffer, // buffer(0)
+ pub keys_buffer: &'a B::Buffer, // buffer(1)
+ pub values_buffer: &'a B::Buffer, // buffer(2)
+ pub output_buffer: &'a mut B::Buffer, // buffer(3)
+ pub mask_buffer: Option<&'a B::Buffer>, // buffer(6)
+ pub sinks_buffer: Option<&'a B::Buffer>, // buffer(7)
pub num_heads: usize,
pub num_groups: usize,
pub suffix_length: usize, // qL
@@ -49,7 +49,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
pub fn encode(
&self,
context: &B::Context,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: AttentionGemmArguments<B>,
) -> Result<(), B::Error> {
let bk: usize = if args.head_dim < 128 {
@@ -138,7 +138,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
args.sinks_buffer,
args.num_heads as u32,
args.suffix_length as u32,
- compute_encoder,
+ command_buffer,
);
Ok(())
diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rs
index dbe4e942f..04692fa70 100644
--- a/crates/uzu/src/backends/common/kernel/audio.rs
+++ b/crates/uzu/src/backends/common/kernel/audio.rs
@@ -1,4 +1,4 @@
-use crate::backends::common::Backend;
+use crate::backends::common::{Backend, CommandBuffer};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AudioPadMode {
@@ -7,9 +7,9 @@ pub enum AudioPadMode {
}
pub struct AudioFsqDecodeArguments<'a, B: Backend> {
- pub tokens: &'a B::NativeBuffer,
- pub lengths: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
+ pub tokens: &'a B::Buffer,
+ pub lengths: &'a B::Buffer,
+ pub output: &'a B::Buffer,
pub batch_size: usize,
pub num_groups: usize,
pub seq_len: usize,
@@ -18,9 +18,9 @@ pub struct AudioFsqDecodeArguments<'a, B: Backend> {
}
pub struct AudioFsqEncodeArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub tokens: &'a B::NativeBuffer,
- pub lengths: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub tokens: &'a B::Buffer,
+ pub lengths: &'a B::Buffer,
pub batch_size: usize,
pub num_groups: usize,
pub seq_len: usize,
@@ -31,11 +31,11 @@ pub struct AudioFsqEncodeArguments<'a, B: Backend> {
}
pub struct AudioConv1dArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub weight: &'a B::NativeBuffer,
- pub bias: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
- pub lengths: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub weight: &'a B::Buffer,
+ pub bias: &'a B::Buffer,
+ pub output: &'a B::Buffer,
+ pub lengths: &'a B::Buffer,
pub batch_size: usize,
pub cin: usize,
pub cout: usize,
@@ -49,11 +49,11 @@ pub struct AudioConv1dArguments<'a, B: Backend> {
}
pub struct AudioCausalConv1dArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub weight: &'a B::NativeBuffer,
- pub bias: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
- pub lengths: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub weight: &'a B::Buffer,
+ pub bias: &'a B::Buffer,
+ pub output: &'a B::Buffer,
+ pub lengths: &'a B::Buffer,
pub batch_size: usize,
pub cin: usize,
pub cout: usize,
@@ -63,11 +63,11 @@ pub struct AudioCausalConv1dArguments<'a, B: Backend> {
}
pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub weight: &'a B::NativeBuffer,
- pub bias: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
- pub lengths: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub weight: &'a B::Buffer,
+ pub bias: &'a B::Buffer,
+ pub output: &'a B::Buffer,
+ pub lengths: &'a B::Buffer,
pub batch_size: usize,
pub cin: usize,
pub cout: usize,
@@ -78,9 +78,9 @@ pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {
}
pub struct AudioHalfSnakeArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub alpha: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub alpha: &'a B::Buffer,
+ pub output: &'a B::Buffer,
pub batch_size: usize,
pub channels: usize,
pub seq_len: usize,
@@ -90,28 +90,28 @@ pub struct AudioHalfSnakeArguments<'a, B: Backend> {
}
pub struct AudioElementwiseArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub output: &'a B::Buffer,
pub n: usize,
}
pub struct AudioAddArguments<'a, B: Backend> {
- pub a: &'a B::NativeBuffer,
- pub b: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
+ pub a: &'a B::Buffer,
+ pub b: &'a B::Buffer,
+ pub output: &'a B::Buffer,
pub n: usize,
}
pub struct AudioScaleArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub output: &'a B::Buffer,
pub n: usize,
pub scale: f32,
}
pub struct AudioClampArguments<'a, B: Backend> {
- pub input: &'a B::NativeBuffer,
- pub output: &'a B::NativeBuffer,
+ pub input: &'a B::Buffer,
+ pub output: &'a B::Buffer,
pub n: usize,
pub min_value: f32,
pub max_value: f32,
@@ -120,68 +120,68 @@ pub struct AudioClampArguments<'a, B: Backend> {
pub trait AudioKernelRuntime<B: Backend> {
fn encode_fsq_decode(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioFsqDecodeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_fsq_encode(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioFsqEncodeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_conv1d(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioConv1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_causal_conv1d(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioCausalConv1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_causal_conv_transpose1d(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioCausalConvTranspose1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_half_snake(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioHalfSnakeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_leaky_relu(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioElementwiseArguments<'_, B>,
negative_slope: f32,
) -> Result<(), B::Error>;
fn encode_tanh(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioElementwiseArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_add(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioAddArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_scale(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioScaleArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_clamp(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: AudioClampArguments<'_, B>,
) -> Result<(), B::Error>;
}
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
index bf6a43bae..def3ce934 100644
--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -8,9 +8,9 @@ use crate::{
};
pub struct KVLayerData<B: Backend> {
- pub key_buffer: Rc<RefCell<B::NativeBuffer>>,
+ pub key_buffer: Rc<RefCell<B::Buffer>>,
pub key_shape: [usize; 3],
- pub value_buffer: Rc<RefCell<B::NativeBuffer>>,
+ pub value_buffer: Rc<RefCell<B::Buffer>>,
pub value_shape: [usize; 3],
}
@@ -52,25 +52,13 @@ impl<B: Backend> KVCacheUpdate<B> {
})
}
+ /// Encode the KV cache update operation using a provided command buffer
pub fn encode(
&self,
in_place_data: &[KVLayerData<B>],
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &mut B::CommandBuffer,
- ) -> Result<(), KVCacheUpdateError<B>> {
- command_buffer.with_compute_encoder(|encoder| {
- self.encode_with_encoder(in_place_data, source_indices, destination_indices, encoder)
- })
- }
-
- /// Encode the KV cache update operation using a provided compute encoder
- pub fn encode_with_encoder(
- &self,
- in_place_data: &[KVLayerData<B>],
- source_indices: &[usize],
- destination_indices: &[usize],
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), KVCacheUpdateError<B>> {
if source_indices.len() != destination_indices.len() {
return Err(KVCacheUpdateError::IndicesCountMismatch);
@@ -99,7 +87,7 @@ impl<B: Backend> KVCacheUpdate<B> {
num_heads as u32,
max_sequence_length as u32,
head_dim as u32,
- encoder,
+ command_buffer,
);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
index ad3abda7f..ec91a6d71 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
@@ -1,12 +1,15 @@
use super::{MatmulError, MatmulKernels};
-use crate::{DataType, backends::common::Backend};
+use crate::{
+ DataType,
+ backends::common::{Backend, CommandBuffer},
+};
pub struct FullPrecisionMatmulArguments<'a, B: Backend> {
- pub a: &'a B::NativeBuffer,
+ pub a: &'a B::Buffer,
pub a_offset: usize,
- pub b: &'a B::NativeBuffer,
- pub output: &'a mut B::NativeBuffer,
- pub bias: Option<&'a B::NativeBuffer>,
+ pub b: &'a B::Buffer,
+ pub output: &'a mut B::Buffer,
+ pub bias: Option<&'a B::Buffer>,
pub batch: usize,
pub input_dim: usize,
pub output_dim: usize,
@@ -23,7 +26,7 @@ pub trait FullPrecisionMatmulKernel: Sized {
fn encode(
&mut self,
context: &<Self::Backend as Backend>::Context,
- encoder: &mut <Self::Backend as Backend>::ComputeEncoder,
+ command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
arguments: FullPrecisionMatmulArguments<Self::Backend>,
);
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
index 4c78c8d84..ca3df3b48 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
@@ -6,7 +6,7 @@ use super::{
use crate::{
DataType,
backends::common::{
- Backend, Kernels,
+ Backend, CommandBuffer, Kernels,
kernel::{MatmulGemmKernel, matmul::MatmulError},
},
};
@@ -66,7 +66,7 @@ impl<B: Backend> GemmKernel<B> {
context: &B::Context,
arguments: &mut MatmulArguments<B>,
dispatch_descriptor: &DispatchDescriptor,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
let config = dispatch_descriptor.specialization;
@@ -86,7 +86,7 @@ impl<B: Backend> GemmKernel<B> {
group_count_x,
group_count_y,
group_count_z,
- encoder,
+ command_buffer,
);
Ok(())
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
index 2a3a83c15..e4046f267 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
@@ -8,7 +8,7 @@ use super::{
use crate::{
DataType,
backends::common::{
- Backend, Kernels,
+ Backend, CommandBuffer, Kernels,
kernel::{MatmulGemvKernel, matmul::MatmulError},
},
};
@@ -67,7 +67,7 @@ impl<B: Backend> GemvKernel<B> {
context: &B::Context,
arguments: &mut MatmulArguments<B>,
dispatch_descriptor: &DispatchDescriptor,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
let config = dispatch_descriptor.specialization;
let pipeline = self.get_or_create_kernel(context, config)?;
@@ -118,7 +118,7 @@ impl<B: Backend> GemvKernel<B> {
dispatch_descriptor.bias_stride,
dispatch_descriptor.batch_rows,
config.output_rows_per_threadgroup() as i32,
- encoder,
+ command_buffer,
);
Ok(())
diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
index c1c8d0954..e021bf13c 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -6,7 +6,7 @@ use super::{
};
use crate::{
DataType,
- backends::common::{Backend, Kernels, kernel::TensorAddBiasKernel},
+ backends::common::{Backend, CommandBuffer, Kernels, kernel::TensorAddBiasKernel},
};
pub struct MatmulKernel<B: Backend> {
@@ -74,20 +74,20 @@ impl<B: Backend> MatmulKernel<B> {
context: &B::Context,
arguments: &mut MatmulArguments<B>,
dispatch_descriptor: &MatmulDispatchDescriptor,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
match dispatch_descriptor {
MatmulDispatchDescriptor::Gemv(d) => {
let gemv = self.get_or_create_gemv()?;
- gemv.encode(context, arguments, d, encoder)
+ gemv.encode(context, arguments, d, command_buffer)
},
MatmulDispatchDescriptor::SplitK(d) => {
let splitk = self.get_or_create_splitk()?;
- splitk.encode(context, arguments, d, encoder)
+ splitk.encode(context, arguments, d, command_buffer)
},
MatmulDispatchDescriptor::Gemm(d) => {
let gemm = self.get_or_create_gemm()?;
- gemm.encode(context, arguments, d, encoder)
+ gemm.encode(context, arguments, d, command_buffer)
},
}
}
@@ -97,13 +97,13 @@ impl<B: Backend> MatmulKernel<B> {
context: &B::Context,
mut arguments: MatmulArguments<B>,
dispatch_descriptor: &MatmulDispatchDescriptor,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
- self.encode_dispatch_descriptor(context, &mut arguments, dispatch_descriptor, encoder)?;
+ self.encode_dispatch_descriptor(context, &mut arguments, dispatch_descriptor, command_buffer)?;
if let Some(bias) = arguments.bias {
if !dispatch_descriptor.bias_is_fused() {
- self.apply_bias_add(context, &mut arguments, bias, encoder)?;
+ self.apply_bias_add(context, &mut arguments, bias, command_buffer)?;
}
}
@@ -114,8 +114,8 @@ impl<B: Backend> MatmulKernel<B> {
&mut self,
context: &B::Context,
arguments: &mut MatmulArguments<B>,
- bias: &B::NativeBuffer,
- encoder: &mut B::ComputeEncoder,
+ bias: &B::Buffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
let m = arguments.batch as usize;
let n = arguments.output_dim as usize;
@@ -132,7 +132,7 @@ impl<B: Backend> MatmulKernel<B> {
);
}
let bias_add = self.bias_add.as_ref().unwrap();
- bias_add.encode(None::<&B::NativeBuffer>, bias, arguments.d.deref_mut(), n as u32, total_len as u32, encoder);
+ bias_add.encode(None::<&B::Buffer>, bias, arguments.d.deref_mut(), n as u32, total_len as u32, command_buffer);
Ok(())
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
index ce4e478c1..49c60a860 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
@@ -2,12 +2,12 @@ use crate::backends::common::Backend;
#[derive(Debug)]
pub struct MatmulArguments<'a, B: Backend> {
- pub a: &'a B::NativeBuffer,
+ pub a: &'a B::Buffer,
/// Byte offset into `a` (used for slicing the batch dimension).
pub a_offset: u64,
- pub b: &'a B::NativeBuffer,
- pub d: &'a mut B::NativeBuffer,
- pub bias: Option<&'a B::NativeBuffer>,
+ pub b: &'a B::Buffer,
+ pub d: &'a mut B::Buffer,
+ pub bias: Option<&'a B::Buffer>,
/// M dimension - batch/number of tokens (rows of A, rows of D)
pub batch: i32,
/// K dimension - input_dim/reduction dimension (cols of A, rows of B)
diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
index 0116066c1..08cd9b9d2 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
@@ -4,7 +4,7 @@ use super::{super::matmul_arguments::MatmulArguments, dispatch_descriptor::Dispa
use crate::{
DataType,
backends::common::{
- Backend, Context, Kernels,
+ Backend, CommandBuffer, Context, Kernels,
kernel::{MatmulSplitKAccumBfloat16Kernel, MatmulSplitKPartialBfloat16Kernel, matmul::MatmulError},
},
};
@@ -13,7 +13,7 @@ pub struct SplitKKernel<B: Backend> {
data_type: DataType,
partial_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKPartialBfloat16Kernel>,
accum_bfloat16: Option<<B::Kernels as Kernels>::MatmulSplitKAccumBfloat16Kernel>,
- accumulator_buffer: Option<B::NativeBuffer>,
+ accumulator_buffer: Option<B::Buffer>,
accumulator_buffer_bytes: usize,
}
@@ -68,7 +68,7 @@ impl<B: Backend> SplitKKernel<B> {
context: &B::Context,
arguments: &mut MatmulArguments<B>,
dispatch_descriptor: &DispatchDescriptor,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), MatmulError<B>> {
self.ensure_kernels(context)?;
self.ensure_accumulator_buffer(context, dispatch_descriptor.accumulator_bytes)?;
@@ -89,7 +89,7 @@ impl<B: Backend> SplitKKernel<B> {
partial_group_count_x,
partial_group_count_y,
partial_group_count_z,
- encoder,
+ command_buffer,
);
let accum_total_threads_x = u32::try_from(dispatch_descriptor.accum_total_threads.x)
@@ -105,7 +105,7 @@ impl<B: Backend> SplitKKernel<B> {
arguments.ldd,
accum_total_threads_x,
accum_total_threads_y,
- encoder,
+ command_buffer,
);
Ok(())
diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
index 3e56c2318..6e83fefa2 100644
--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -1,6 +1,6 @@
use crate::{
DataType,
- backends::common::{Backend, Kernels, kernel::MlpGateActMulKernel},
+ backends::common::{Backend, CommandBuffer, Kernels, kernel::MlpGateActMulKernel},
config::Activation,
};
@@ -27,9 +27,9 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
pub fn encode(
&self,
- encoder: &mut B::ComputeEncoder,
- fused_up: &B::NativeBuffer,
- hidden: &mut B::NativeBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ fused_up: &B::Buffer,
+ hidden: &mut B::Buffer,
m: i32,
) -> Result<(), B::Error> {
let act_type = match self.activation {
@@ -41,7 +41,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
panic!("Identity activation is not supported for kernel")
},
};
- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);
+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, command_buffer);
Ok(())
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
index f2d3d84f0..bcaa38a61 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
@@ -14,23 +14,23 @@ static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
#[derive(Debug)]
pub struct MoeExpertsSingleDecodeArguments<'a, B: Backend> {
/// Input activation [d_model]
- pub x: &'a B::NativeBuffer,
+ pub x: &'a B::Buffer,
/// Top-K expert indices from router [K]
- pub topk_ids: &'a B::NativeBuffer,
+ pub topk_ids: &'a B::Buffer,
/// Top-K probabilities from router [K]
- pub topk_probs: &'a B::NativeBuffer,
+ pub topk_probs: &'a B::Buffer,
/// Up/gate projection weights [E, 2*d_ff, d_model]
- pub w13_all: &'a B::NativeBuffer,
+ pub w13_all: &'a B::Buffer,
/// Down projection weights [E, d_model, d_ff]
- pub w2_all: &'a B::NativeBuffer,
+ pub w2_all: &'a B::Buffer,
/// Up/gate biases [E, 2*d_ff]
- pub up_biases: &'a B::NativeBuffer,
+ pub up_biases: &'a B::Buffer,
/// Down biases [E, d_model]
- pub down_biases: &'a B::NativeBuffer,
+ pub down_biases: &'a B::Buffer,
/// Hidden buffer [K, d_ff] - intermediate storage (f32)
- pub hidden: &'a mut B::NativeBuffer,
+ pub hidden: &'a mut B::Buffer,
/// Final output [d_model]
- pub y: &'a mut B::NativeBuffer,
+ pub y: &'a mut B::Buffer,
/// Model dimension
pub d_model: usize,
/// FFN hidden dimension
@@ -84,7 +84,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
pub fn encode(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
mut args: MoeExpertsSingleDecodeArguments<B>,
) {
if args.k == 0 {
@@ -95,41 +95,37 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
// Pass A: x @ W13[expert] -> hidden
- command_buffer.with_compute_encoder(|encoder| {
- let kernel = &self.pass_a[gate_idx][dtype_idx];
- kernel.encode(
- args.x,
- args.topk_ids,
- args.w13_all,
- args.up_biases,
- args.hidden.deref_mut(),
- args.d_model as u32,
- args.d_ff as u32,
- args.k as u32,
- args.silu_alpha,
- args.gate_clip_min,
- args.gate_clip_max,
- args.up_clip_min,
- args.up_clip_max,
- encoder,
- );
- });
+ let kernel = &self.pass_a[gate_idx][dtype_idx];
+ kernel.encode(
+ args.x,
+ args.topk_ids,
+ args.w13_all,
+ args.up_biases,
+ args.hidden.deref_mut(),
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.k as u32,
+ args.silu_alpha,
+ args.gate_clip_min,
+ args.gate_clip_max,
+ args.up_clip_min,
+ args.up_clip_max,
+ command_buffer,
+ );
// Pass B: 8 simdgroups (256 threads), outputs final y directly
- command_buffer.with_compute_encoder(|encoder| {
- let kernel = &self.pass_b[dtype_idx];
- kernel.encode(
- args.hidden.deref(),
- args.topk_ids,
- args.topk_probs,
- args.w2_all,
- args.down_biases,
- args.y,
- args.d_model as u32,
- args.d_ff as u32,
- args.k as u32,
- encoder,
- );
- });
+ let kernel = &self.pass_b[dtype_idx];
+ kernel.encode(
+ args.hidden.deref(),
+ args.topk_ids,
+ args.topk_probs,
+ args.w2_all,
+ args.down_biases,
+ args.y,
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.k as u32,
+ command_buffer,
+ );
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
index 7b7adfcde..1030d6a5c 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
@@ -7,8 +7,8 @@ use super::{
use crate::{
DataType,
backends::common::{
- CommandBuffer,
- kernel::{Backend, Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},
+ Backend, CommandBuffer,
+ kernel::{Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},
},
};
@@ -47,7 +47,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
pub fn encode(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
mut args: MoeExpertsTwoPassArguments<B>,
) {
if args.total_rows == 0 {
@@ -108,43 +108,39 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
let dtype_idx = DTYPES.iter().position(|dtype| *dtype == args.data_type).unwrap();
// pass a
- command_buffer.with_compute_encoder(|encoder| {
- let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
- pass_a_kernel.encode(
- args.x_perm_buffer,
- args.expert_offsets,
- args.w13_all,
- args.hidden_buffer.deref_mut(),
- args.up_biases,
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- args.gate_clip_min,
- args.gate_clip_max,
- args.up_clip_min,
- args.up_clip_max,
- args.silu_alpha,
- args.tile_map.deref(),
- args.dispatch_args.deref(),
- encoder,
- );
- });
+ let pass_a_kernel = &self.pass_a_indirect[gate_idx][dtype_idx];
+ pass_a_kernel.encode(
+ args.x_perm_buffer,
+ args.expert_offsets,
+ args.w13_all,
+ args.hidden_buffer.deref_mut(),
+ args.up_biases,
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ args.gate_clip_min,
+ args.gate_clip_max,
+ args.up_clip_min,
+ args.up_clip_max,
+ args.silu_alpha,
+ args.tile_map.deref(),
+ args.dispatch_args.deref(),
+ command_buffer,
+ );
// pass b
- command_buffer.with_compute_encoder(|encoder| {
- let pass_b_kernel = &self.fused_down[dtype_idx];
- pass_b_kernel.encode(
- args.hidden_buffer.deref(),
- args.row_expert_map.deref(),
- args.w2_all,
- args.down_biases,
- args.output_buffer.deref_mut(),
- args.total_rows as u32,
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- encoder,
- );
- });
+ let pass_b_kernel = &self.fused_down[dtype_idx];
+ pass_b_kernel.encode(
+ args.hidden_buffer.deref(),
+ args.row_expert_map.deref(),
+ args.w2_all,
+ args.down_biases,
+ args.output_buffer.deref_mut(),
+ args.total_rows as u32,
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ command_buffer,
+ );
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
index 10c3f74d6..15cd2e280 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
@@ -6,7 +6,7 @@ use super::{
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, CopyEncoder, Kernels,
+ Backend, CommandBuffer, CommandBufferEncoding, Kernels,
kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},
},
};
@@ -48,7 +48,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
pub fn encode(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
mut args: MoeExpertsTwoPassArguments<B>,
) {
if args.total_rows == 0 {
@@ -56,8 +56,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
}
let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
- command_buffer
- .with_copy_encoder(|encoder| encoder.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0));
+ command_buffer.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);
self.tile_map.encode_counts(
command_buffer,
@@ -112,27 +111,25 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
let gate_idx = args.gating_code.min(3) as usize;
let dtype_idx = DTYPES.iter().position(|t| *t == args.data_type).unwrap();
- command_buffer.with_compute_encoder(|encoder| {
- let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];
- kernel_pass_a.encode(
- args.x_perm_buffer,
- args.expert_offsets,
- args.w13_all,
- args.up_biases,
- args.hidden_buffer.deref_mut(),
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- args.gate_clip_min,
- args.gate_clip_max,
- args.up_clip_min,
- args.up_clip_max,
- args.silu_alpha,
- args.tile_map.deref(),
- args.dispatch_args.deref(),
- encoder,
- );
- });
+ let kernel_pass_a = &self.pass_a_indirect[gate_idx][dtype_idx];
+ kernel_pass_a.encode(
+ args.x_perm_buffer,
+ args.expert_offsets,
+ args.w13_all,
+ args.up_biases,
+ args.hidden_buffer.deref_mut(),
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ args.gate_clip_min,
+ args.gate_clip_max,
+ args.up_clip_min,
+ args.up_clip_max,
+ args.silu_alpha,
+ args.tile_map.deref(),
+ args.dispatch_args.deref(),
+ command_buffer,
+ );
let dispatch_args = MoeTileDispatchArguments {
total_tiles: args.total_tiles,
@@ -141,41 +138,39 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
};
self.tile_map.encode_dispatch_args(command_buffer, dispatch_args);
- command_buffer.with_compute_encoder(|encoder| {
- let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
- kernel_pass_b.encode(
- args.hidden_buffer.deref(),
- args.expert_offsets,
- args.w2_all,
- args.down_biases,
- args.output_buffer.deref_mut(),
- args.d_model as u32,
- args.d_ff as u32,
- args.e as u32,
- args.tile_map.deref(),
- args.dispatch_args.deref(),
- encoder,
- );
- });
+ let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
+ kernel_pass_b.encode(
+ args.hidden_buffer.deref(),
+ args.expert_offsets,
+ args.w2_all,
+ args.down_biases,
+ args.output_buffer.deref_mut(),
+ args.d_model as u32,
+ args.d_ff as u32,
+ args.e as u32,
+ args.tile_map.deref(),
+ args.dispatch_args.deref(),
+ command_buffer,
+ );
}
}
#[derive(Debug)]
pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {
- pub x_perm_buffer: &'a B::NativeBuffer,
- pub expert_offsets: &'a B::NativeBuffer,
- pub row_expert_map: &'a mut B::NativeBuffer,
- pub hidden_buffer: &'a mut B::NativeBuffer,
- pub output_buffer: &'a mut B::NativeBuffer,
- pub w13_all: &'a B::NativeBuffer,
- pub w2_all: &'a B::NativeBuffer,
- pub up_biases: &'a B::NativeBuffer,
- pub down_biases: &'a B::NativeBuffer,
- pub tile_counts: &'a mut B::NativeBuffer,
- pub tile_offsets: &'a mut B::NativeBuffer,
- pub tile_map: &'a mut B::NativeBuffer,
- pub total_tiles: &'a mut B::NativeBuffer,
- pub dispatch_args: &'a mut B::NativeBuffer,
+ pub x_perm_buffer: &'a B::Buffer,
+ pub expert_offsets: &'a B::Buffer,
+ pub row_expert_map: &'a mut B::Buffer,
+ pub hidden_buffer: &'a mut B::Buffer,
+ pub output_buffer: &'a mut B::Buffer,
+ pub w13_all: &'a B::Buffer,
+ pub w2_all: &'a B::Buffer,
+ pub up_biases: &'a B::Buffer,
+ pub down_biases: &'a B::Buffer,
+ pub tile_counts: &'a mut B::Buffer,
+ pub tile_offsets: &'a mut B::Buffer,
+ pub tile_map: &'a mut B::Buffer,
+ pub total_tiles: &'a mut B::Buffer,
+ pub dispatch_args: &'a mut B::Buffer,
pub total_rows: usize,
pub d_model: usize,
pub d_ff: usize,
diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rs
index 38a4dd545..771c32ada 100644
--- a/crates/uzu/src/backends/common/kernel/moe/gather.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs
@@ -8,10 +8,10 @@ use crate::{
#[derive(Debug)]
pub struct MoeGatherArguments<'a, B: Backend> {
- pub x_buffer: &'a B::NativeBuffer,
- pub bucketed_ids_buffer: &'a B::NativeBuffer,
- pub x_perm_buffer: &'a mut B::NativeBuffer,
- pub sumk_buffer: &'a B::NativeBuffer,
+ pub x_buffer: &'a B::Buffer,
+ pub bucketed_ids_buffer: &'a B::Buffer,
+ pub x_perm_buffer: &'a mut B::Buffer,
+ pub sumk_buffer: &'a B::Buffer,
pub t: usize,
pub k: usize,
pub d_model: usize,
@@ -34,11 +34,11 @@ impl<B: Backend> MoeGatherKernels<B> {
pub fn encode(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
dtype: DataType,
args: MoeGatherArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| match dtype {
+ match dtype {
DataType::F32 => self.f32.encode(
args.x_buffer,
args.bucketed_ids_buffer,
@@ -47,7 +47,7 @@ impl<B: Backend> MoeGatherKernels<B> {
args.d_model as u32,
args.t as u32,
args.k as u32,
- encoder,
+ command_buffer,
),
DataType::F16 => self.f16.encode(
args.x_buffer,
@@ -57,7 +57,7 @@ impl<B: Backend> MoeGatherKernels<B> {
args.d_model as u32,
args.t as u32,
args.k as u32,
- encoder,
+ command_buffer,
),
DataType::BF16 => self.bf16.encode(
args.x_buffer,
@@ -67,9 +67,9 @@ impl<B: Backend> MoeGatherKernels<B> {
args.d_model as u32,
args.t as u32,
args.k as u32,
- encoder,
+ command_buffer,
),
_ => panic!("Unsupported data type: {:?}", dtype),
- });
+ };
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
index fb5712cd2..ea144b317 100644
--- a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
@@ -22,83 +22,75 @@ impl<B: Backend> MoeTileMapKernels<B> {
pub fn encode_counts(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoeTileCountsArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);
- });
+ self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, command_buffer);
}
pub fn encode_scan(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoeTileScanArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.scan.encode(
- args.tile_counts_buffer,
- args.tile_offsets_buffer,
- args.total_tiles_buffer,
- args.e as u32,
- encoder,
- );
- });
+ self.scan.encode(
+ args.tile_counts_buffer,
+ args.tile_offsets_buffer,
+ args.total_tiles_buffer,
+ args.e as u32,
+ command_buffer,
+ );
}
pub fn encode_build_map(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoeTileMapBuildArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.build.encode(
- args.expert_offsets,
- args.tile_offsets,
- args.tile_counts,
- args.tile_map,
- args.e as u32,
- encoder,
- );
- });
+ self.build.encode(
+ args.expert_offsets,
+ args.tile_offsets,
+ args.tile_counts,
+ args.tile_map,
+ args.e as u32,
+ command_buffer,
+ );
}
pub fn encode_dispatch_args(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoeTileDispatchArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);
- });
+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, command_buffer);
}
}
pub struct MoeTileCountsArguments<'a, B: Backend> {
- pub offsets_buffer: &'a B::NativeBuffer, // [E+1]
- pub tile_counts_buffer: &'a mut B::NativeBuffer, // [E]
+ pub offsets_buffer: &'a B::Buffer, // [E+1]
+ pub tile_counts_buffer: &'a mut B::Buffer, // [E]
pub e: usize,
}
pub struct MoeTileScanArguments<'a, B: Backend> {
- pub tile_counts_buffer: &'a B::NativeBuffer, // [E]
- pub tile_offsets_buffer: &'a mut B::NativeBuffer, // [E+1]
- pub total_tiles_buffer: &'a mut B::NativeBuffer, // [>=2]
+ pub tile_counts_buffer: &'a B::Buffer, // [E]
+ pub tile_offsets_buffer: &'a mut B::Buffer, // [E+1]
+ pub total_tiles_buffer: &'a mut B::Buffer, // [>=2]
pub e: usize,
}
#[derive(Debug)]
pub struct MoeTileMapBuildArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::NativeBuffer, // [E+1]
- pub tile_offsets: &'a B::NativeBuffer, // [E+1]
- pub tile_counts: &'a B::NativeBuffer, // [E]
- pub tile_map: &'a mut B::NativeBuffer, // [total_tiles * 3]
+ pub expert_offsets: &'a B::Buffer, // [E+1]
+ pub tile_offsets: &'a B::Buffer, // [E+1]
+ pub tile_counts: &'a B::Buffer, // [E]
+ pub tile_map: &'a mut B::Buffer, // [total_tiles * 3]
pub e: usize,
}
#[derive(Debug)]
pub struct MoeTileDispatchArguments<'a, B: Backend> {
- pub total_tiles: &'a B::NativeBuffer, // [>=1]
- pub dispatch_args: &'a mut B::NativeBuffer, // [3]
- pub num_tiles_x: u32, // x dimension for indirect dispatch
+ pub total_tiles: &'a B::Buffer, // [>=1]
+ pub dispatch_args: &'a mut B::Buffer, // [3]
+ pub num_tiles_x: u32, // x dimension for indirect dispatch
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
index 8704a7e03..22353c53c 100644
--- a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
@@ -1,49 +1,49 @@
use crate::backends::common::{
- CommandBuffer,
+ Backend, CommandBuffer,
kernel::{
- Backend, Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,
+ Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,
MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,
},
};
#[derive(Debug)]
pub struct MoePassATileCountsArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::NativeBuffer, // [E+1]
- pub tile_counts: &'a mut B::NativeBuffer, // [E]
+ pub expert_offsets: &'a B::Buffer, // [E+1]
+ pub tile_counts: &'a mut B::Buffer, // [E]
pub e: usize,
pub h_blocks: u32,
}
#[derive(Debug)]
pub struct MoePassATileScanArguments<'a, B: Backend> {
- pub tile_counts: &'a B::NativeBuffer, // [E]
- pub tile_offsets: &'a mut B::NativeBuffer, // [E+1]
- pub total_tiles: &'a mut B::NativeBuffer, // [>=1]
+ pub tile_counts: &'a B::Buffer, // [E]
+ pub tile_offsets: &'a mut B::Buffer, // [E+1]
+ pub total_tiles: &'a mut B::Buffer, // [>=1]
pub e: usize,
}
#[derive(Debug)]
pub struct MoePassARowMapArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::NativeBuffer, // [E+1]
- pub row_expert_map: &'a mut B::NativeBuffer, // [total_rows]
+ pub expert_offsets: &'a B::Buffer, // [E+1]
+ pub row_expert_map: &'a mut B::Buffer, // [total_rows]
pub total_rows: usize,
pub e: usize,
}
#[derive(Debug)]
pub struct MoePassATileBuildArguments<'a, B: Backend> {
- pub expert_offsets: &'a B::NativeBuffer, // [E+1]
- pub tile_offsets: &'a B::NativeBuffer, // [E+1]
- pub row_expert_map: &'a B::NativeBuffer, // [total_rows]
- pub tile_map: &'a mut B::NativeBuffer, // [total_tiles * 3]
+ pub expert_offsets: &'a B::Buffer, // [E+1]
+ pub tile_offsets: &'a B::Buffer, // [E+1]
+ pub row_expert_map: &'a B::Buffer, // [total_rows]
+ pub tile_map: &'a mut B::Buffer, // [total_tiles * 3]
pub total_rows: usize,
pub h_blocks: u32,
}
#[derive(Debug)]
pub struct MoePassATileDispatchArguments<'a, B: Backend> {
- pub total_tiles: &'a B::NativeBuffer, // [>=1]
- pub dispatch_args: &'a mut B::NativeBuffer, // [3]
+ pub total_tiles: &'a B::Buffer, // [>=1]
+ pub dispatch_args: &'a mut B::Buffer, // [3]
pub num_tiles_y: u32,
}
@@ -68,65 +68,55 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_counts(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoePassATileCountsArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);
- });
+ self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, command_buffer);
}
pub fn encode_scan(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoePassATileScanArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);
- });
+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, command_buffer);
}
pub fn encode_row_map(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoePassARowMapArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.row_map.encode(
- args.expert_offsets,
- args.row_expert_map,
- args.total_rows as u32,
- args.e as u32,
- encoder,
- );
- });
+ self.row_map.encode(
+ args.expert_offsets,
+ args.row_expert_map,
+ args.total_rows as u32,
+ args.e as u32,
+ command_buffer,
+ );
}
pub fn encode_build_map(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoePassATileBuildArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.build_map.encode(
- args.expert_offsets,
- args.tile_offsets,
- args.row_expert_map,
- args.tile_map,
- args.total_rows as u32,
- args.h_blocks,
- encoder,
- );
- });
+ self.build_map.encode(
+ args.expert_offsets,
+ args.tile_offsets,
+ args.row_expert_map,
+ args.tile_map,
+ args.total_rows as u32,
+ args.h_blocks,
+ command_buffer,
+ );
}
pub fn encode_dispatch_args(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: MoePassATileDispatchArguments<B>,
) {
- command_buffer.with_compute_encoder(|encoder| {
- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);
- });
+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, command_buffer);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index 6358aa83f..40daa27df 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -3,7 +3,7 @@ use std::collections::HashMap;
use crate::{
DataType,
backends::common::{
- Backend, Kernels,
+ Backend, CommandBuffer, Kernels,
kernel::{
QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,
@@ -54,12 +54,12 @@ pub struct QuantizedMatmulConfiguration {
}
pub struct QuantizedMatmulArguments<'a, B: Backend> {
- pub a_buffer: &'a B::NativeBuffer,
+ pub a_buffer: &'a B::Buffer,
pub a_offset: usize,
- pub b_buffer: &'a B::NativeBuffer,
- pub scales_buffer: &'a B::NativeBuffer,
- pub zero_points_or_biases_buffer: &'a B::NativeBuffer,
- pub output_buffer: &'a mut B::NativeBuffer,
+ pub b_buffer: &'a B::Buffer,
+ pub scales_buffer: &'a B::Buffer,
+ pub zero_points_or_biases_buffer: &'a B::Buffer,
+ pub output_buffer: &'a mut B::Buffer,
pub batch: usize,
pub input_dim: usize,
pub output_dim: usize,
@@ -161,7 +161,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
pub fn encode(
&self,
- encoder: &mut B::ComputeEncoder,
+ encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
arguments: QuantizedMatmulArguments<B>,
) -> Result<(), QuantizedMatmulError<B>> {
if arguments.quantization_type != self.quantization_type {
@@ -487,9 +487,9 @@ fn to_i32<B: Backend>(
}
fn quant_buffers<'a, B: Backend>(
- buffer: &'a B::NativeBuffer,
+ buffer: &'a B::Buffer,
quantization_type: QuantizedMatmulType,
-) -> (Option<&'a B::NativeBuffer>, Option<&'a B::NativeBuffer>) {
+) -> (Option<&'a B::Buffer>, Option<&'a B::Buffer>) {
match quantization_type {
QuantizedMatmulType::ZeroPoint => (Some(buffer), None),
QuantizedMatmulType::Mlx => (None, Some(buffer)),
diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rs
index 4a651fd50..921b4b5a9 100644
--- a/crates/uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/uzu/src/backends/common/kernel/sampling.rs
@@ -6,11 +6,10 @@ use std::{
use thiserror::Error;
-use super::{Backend, Kernels};
use crate::{
DataType,
backends::common::{
- Context,
+ Backend, CommandBuffer, Context, Kernels,
gpu_types::ArgmaxPair,
kernel::{
ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,
@@ -33,7 +32,7 @@ enum ArgmaxImplementation<B: Backend> {
TwoPass {
main_kernel: <B::Kernels as Kernels>::ArgmaxMainKernel,
final_kernel: <B::Kernels as Kernels>::ArgmaxFinalKernel,
- partial_results_buffer: RefCell<B::NativeBuffer>,
+ partial_results_buffer: RefCell<B::Buffer>,
},
}
@@ -142,18 +141,18 @@ impl<B: Backend> SamplingKernel<B> {
})
}
- pub fn encode_with_encoder(
+ pub fn encode(
&self,
- mut logits_buffer: &mut B::NativeBuffer,
- seeds_buffer: Option<&B::NativeBuffer>,
+ mut logits_buffer: &mut B::Buffer,
+ seeds_buffer: Option<&B::Buffer>,
seeds_offset: usize,
- bitmask_buffer: Option<&B::NativeBuffer>,
+ bitmask_buffer: Option<&B::Buffer>,
bitmask_offset: usize,
- sampled_tokens_buffer: &mut B::NativeBuffer,
+ sampled_tokens_buffer: &mut B::Buffer,
sampling_method: SamplingMethod,
batch_size: usize,
vocab_size: usize,
- compute_encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), SamplingError<B>> {
if batch_size > self.max_batch_size {
return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));
@@ -164,12 +163,12 @@ impl<B: Backend> SamplingKernel<B> {
if let Some(bitmask_buffer) = bitmask_buffer {
self.bitmask.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
(bitmask_buffer, bitmask_offset),
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
- compute_encoder,
+ command_buffer,
);
}
@@ -182,55 +181,55 @@ impl<B: Backend> SamplingKernel<B> {
{
if let Some(temperature) = temperature {
self.temperature.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
temperature,
- compute_encoder,
+ command_buffer,
);
}
if let Some(top_k) = top_k {
self.topk.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
top_k,
- compute_encoder,
+ command_buffer,
);
}
if let Some(top_p) = top_p {
self.topp.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
top_p,
- compute_encoder,
+ command_buffer,
);
}
if let Some(min_p) = min_p {
self.minp.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
min_p,
- compute_encoder,
+ command_buffer,
);
}
self.gumbel.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
(seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
- compute_encoder,
+ command_buffer,
);
}
@@ -243,7 +242,7 @@ impl<B: Backend> SamplingKernel<B> {
sampled_tokens_buffer,
batch_size as u32,
vocab_size as u32,
- compute_encoder,
+ command_buffer,
);
},
ArgmaxImplementation::TwoPass {
@@ -256,14 +255,14 @@ impl<B: Backend> SamplingKernel<B> {
partial_results_buffer.borrow_mut().deref_mut(),
batch_size as u32,
vocab_size as u32,
- compute_encoder,
+ command_buffer,
);
final_kernel.encode(
partial_results_buffer.borrow().deref(),
sampled_tokens_buffer,
batch_size as u32,
vocab_size as u32,
- compute_encoder,
+ command_buffer,
);
},
}
diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
index 20c5da315..cbd8a3f28 100644
--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -1,7 +1,7 @@
use crate::{
DataType,
backends::common::{
- Backend, Kernels,
+ Backend, CommandBuffer, Kernels,
kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},
},
};
@@ -13,14 +13,14 @@ pub enum SSDPrefillMode {
}
pub struct SSDPrefillArguments<'a, B: Backend> {
- pub x: &'a B::NativeBuffer,
- pub dt: &'a B::NativeBuffer, // raw dt values
- pub b: &'a B::NativeBuffer,
- pub c: &'a B::NativeBuffer,
- pub d: &'a B::NativeBuffer,
- pub z: &'a B::NativeBuffer,
- pub state: &'a mut B::NativeBuffer,
- pub y: &'a mut B::NativeBuffer,
+ pub x: &'a B::Buffer,
+ pub dt: &'a B::Buffer, // raw dt values
+ pub b: &'a B::Buffer,
+ pub c: &'a B::Buffer,
+ pub d: &'a B::Buffer,
+ pub z: &'a B::Buffer,
+ pub state: &'a mut B::Buffer,
+ pub y: &'a mut B::Buffer,
pub suffix_len: usize,
pub group_size: i32,
pub state_size: i32,
@@ -55,7 +55,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
pub fn encode(
&self,
- compute_encoder: &mut B::ComputeEncoder,
+ compute_encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
args: SSDPrefillArguments<B>,
mode: SSDPrefillMode,
) {
diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rs
index 9b2caf29a..b98c4bdfa 100644
--- a/crates/uzu/src/backends/common/mod.rs
+++ b/crates/uzu/src/backends/common/mod.rs
@@ -1,22 +1,17 @@
-mod allocator;
mod backend;
mod buffer;
-mod buffer_lifetime;
mod command_buffer;
mod context;
-mod copy_encoder;
mod event;
pub mod gpu_types;
pub mod kernel;
-mod native_buffer;
-pub use allocator::{AllocError, Allocator};
pub use backend::Backend;
pub use buffer::Buffer;
-pub use buffer_lifetime::BufferLifetime;
-pub use command_buffer::CommandBuffer;
+pub use command_buffer::{
+ CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending,
+};
pub use context::{Context, DeviceClass};
-pub use copy_encoder::CopyEncoder;
pub use event::Event;
pub use kernel::Kernels;
-pub use native_buffer::NativeBuffer;
diff --git a/crates/uzu/src/backends/common/native_buffer.rs b/crates/uzu/src/backends/common/native_buffer.rs
deleted file mode 100644
index b5cb6c798..000000000
--- a/crates/uzu/src/backends/common/native_buffer.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};
-
-use super::Backend;
-
-pub trait NativeBuffer: Debug {
- type Backend: Backend;
-
- fn set_label(
- &mut self,
- label: Option<&str>,
- );
-
- fn cpu_ptr(&self) -> NonNull<c_void>;
-
- fn length(&self) -> usize;
-
- fn id(&self) -> usize;
-}
diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rs
index 6ac60bea7..d6a2a5a6e 100644
--- a/crates/uzu/src/backends/cpu/backend.rs
+++ b/crates/uzu/src/backends/cpu/backend.rs
@@ -8,10 +8,8 @@ pub struct Cpu;
impl Backend for Cpu {
type Context = CpuContext;
- type NativeBuffer = Box<[u8]>;
type CommandBuffer = CpuCommandBuffer;
- type ComputeEncoder = CpuCommandBuffer;
- type CopyEncoder = CpuCommandBuffer;
+ type Buffer = Box<[u8]>;
type Event = RefCell<u64>;
type Kernels = CpuKernels;
type Error = CpuError;
diff --git a/crates/uzu/src/backends/cpu/native_buffer.rs b/crates/uzu/src/backends/cpu/buffer.rs
similarity index 84%
rename from crates/uzu/src/backends/cpu/native_buffer.rs
rename to crates/uzu/src/backends/cpu/buffer.rs
index a885455be..306bc05e5 100644
--- a/crates/uzu/src/backends/cpu/native_buffer.rs
+++ b/crates/uzu/src/backends/cpu/buffer.rs
@@ -1,9 +1,9 @@
use std::{os::raw::c_void, ptr::NonNull};
use super::Cpu;
-use crate::backends::common::NativeBuffer;
+use crate::backends::common::Buffer;
-impl NativeBuffer for Box<[u8]> {
+impl Buffer for Box<[u8]> {
type Backend = Cpu;
fn set_label(
diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rs
index 600e880d6..726f09ff6 100644
--- a/crates/uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu/src/backends/cpu/command_buffer.rs
@@ -5,16 +5,29 @@ use std::{
use super::Cpu;
use crate::backends::{
- common::{CommandBuffer, CopyEncoder},
+ common::{
+ CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending,
+ },
cpu::error::CpuError,
};
pub struct CpuCommandBuffer {
commands: Vec<Box<dyn Fn()>>,
- completion_handlers: Vec<Box<dyn FnOnce(Result<(), CpuError>)>>,
+ completion_handlers: Vec<Box<dyn FnOnce(Result<&CpuCommandBuffer, CpuError>)>>,
gpu_execution_time_ms: OnceCell<f64>,
}
+impl CommandBuffer for CpuCommandBuffer {
+ type Backend = Cpu;
+
+ type Initial = CpuCommandBuffer;
+ type Encoding = CpuCommandBuffer;
+ type Executable = CpuCommandBuffer;
+ type Pending = CpuCommandBuffer;
+ type Completed = CpuCommandBuffer;
+}
+
impl CpuCommandBuffer {
pub fn new() -> CpuCommandBuffer {
CpuCommandBuffer {
@@ -32,21 +45,41 @@ impl CpuCommandBuffer {
}
}
-impl CommandBuffer for CpuCommandBuffer {
- type Backend = Cpu;
+impl CommandBufferInitial for CpuCommandBuffer {
+ type CommandBuffer = CpuCommandBuffer;
+
+ fn start_encoding(self) -> CpuCommandBuffer {
+ self
+ }
+}
- fn with_compute_encoder<T>(
+impl CommandBufferEncoding for CpuCommandBuffer {
+ type CommandBuffer = CpuCommandBuffer;
+
+ fn encode_copy(
&mut self,
- callback: impl FnOnce(&mut CpuCommandBuffer) -> T,
- ) -> T {
- callback(self)
+ src: &Box<[u8]>,
+ dst: &mut Box<[u8]>,
+ size: usize,
+ ) {
+ let src = src.as_ptr();
+ let dst = dst.as_ptr() as *mut u8;
+ self.push_command(Box::new(move || unsafe {
+ std::ptr::copy(src, dst, size);
+ }))
}
- fn with_copy_encoder<T>(
+ fn encode_fill(
&mut self,
- callback: impl FnOnce(&mut CpuCommandBuffer) -> T,
- ) -> T {
- callback(self)
+ dst: &mut Box<[u8]>,
+ range: std::ops::Range<usize>,
+ value: u8,
+ ) {
+ let size = range.end - range.start;
+ let dst = dst[range].as_ptr() as *mut u8;
+ self.push_command(Box::new(move || unsafe {
+ dst.write_bytes(value, size);
+ }))
}
fn encode_wait_for_event(
@@ -69,59 +102,53 @@ impl CommandBuffer for CpuCommandBuffer {
fn add_completion_handler(
&mut self,
- handler: impl FnOnce(Result<(), CpuError>) + 'static,
+ handler: impl FnOnce(Result<&CpuCommandBuffer, CpuError>) + 'static,
) {
self.completion_handlers.push(Box::new(handler))
}
- fn submit(&mut self) {
+ fn end_encoding(self) -> CpuCommandBuffer {
+ self
+ }
+}
+
+impl CommandBufferExecutable for CpuCommandBuffer {
+ type CommandBuffer = CpuCommandBuffer;
+
+ fn submit(mut self) -> CpuCommandBuffer {
let start = Instant::now();
for command in &self.commands {
command()
}
+
self.gpu_execution_time_ms
.set((Instant::now() - start).as_secs_f64() * 1000.0)
.expect("gpu execution time already set");
- for completion_handler in self.completion_handlers.drain(..) {
- completion_handler(Ok(()));
- }
- }
- fn wait_until_completed(&self) -> Result<(), CpuError> {
- Ok(())
- }
+ for completion_handler in self.completion_handlers.drain(..).collect::<Vec<_>>() {
+ completion_handler(Ok(&self));
+ }
- fn gpu_execution_time_ms(&self) -> Option<f64> {
- self.gpu_execution_time_ms.get().copied()
+ CpuCommandBuffer {
+ commands: Vec::new(),
+ completion_handlers: Vec::new(),
+ gpu_execution_time_ms: self.gpu_execution_time_ms,
+ }
}
}
-impl CopyEncoder for CpuCommandBuffer {
- type Backend = Cpu;
+impl CommandBufferPending for CpuCommandBuffer {
+ type CommandBuffer = CpuCommandBuffer;
- fn encode_copy(
- &mut self,
- src: &Box<[u8]>,
- dst: &mut Box<[u8]>,
- size: usize,
- ) {
- let src = src.as_ptr();
- let dst = dst.as_ptr() as *mut u8;
- self.push_command(Box::new(move || unsafe {
- std::ptr::copy(src, dst, size);
- }))
+ fn wait_until_completed(self) -> Result<CpuCommandBuffer, CpuError> {
+ Ok(self)
}
+}
- fn encode_fill(
- &mut self,
- dst: &mut Box<[u8]>,
- range: std::ops::Range<usize>,
- value: u8,
- ) {
- let size = range.end - range.start;
- let dst = dst[range].as_ptr() as *mut u8;
- self.push_command(Box::new(move || unsafe {
- dst.write_bytes(value, size);
- }))
+impl CommandBufferCompleted for CpuCommandBuffer {
+ type CommandBuffer = CpuCommandBuffer;
+
+ fn gpu_execution_time_ms(&self) -> Option<f64> {
+ self.gpu_execution_time_ms.get().copied()
}
}
diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rs
index 4efd26ec3..79c450bfc 100644
--- a/crates/uzu/src/backends/cpu/context.rs
+++ b/crates/uzu/src/backends/cpu/context.rs
@@ -1,7 +1,7 @@
use std::{cell::RefCell, rc::Rc};
use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};
-use crate::backends::common::{Allocator, Context, DeviceClass};
+use crate::backends::common::{Context, DeviceClass};
pub struct CpuContext;
@@ -20,10 +20,6 @@ impl Context for CpuContext {
false
}
- fn allocator(&self) -> &Allocator<Self::Backend> {
- unimplemented!()
- }
-
fn create_buffer(
&self,
size: usize,
diff --git a/crates/uzu/src/backends/cpu/mod.rs b/crates/uzu/src/backends/cpu/mod.rs
index bd15afca5..97a2b7485 100644
--- a/crates/uzu/src/backends/cpu/mod.rs
+++ b/crates/uzu/src/backends/cpu/mod.rs
@@ -1,10 +1,10 @@
mod backend;
+mod buffer;
mod command_buffer;
mod context;
mod error;
mod event;
mod kernel;
-mod native_buffer;
pub use backend::Cpu;
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index c8e37bb01..97b583bda 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -1,7 +1,7 @@
-use metal::{MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLComputeCommandEncoder, MTLEvent};
+use metal::{MTLBuffer, MTLEvent};
use objc2::{rc::Retained, runtime::ProtocolObject};
-use super::{context::MetalContext, error::MetalError, kernel::dsl::MetalKernels};
+use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::dsl::MetalKernels};
use crate::backends::common::Backend;
#[derive(Debug, Clone)]
@@ -9,10 +9,8 @@ pub struct Metal;
impl Backend for Metal {
type Context = MetalContext;
- type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;
- type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;
- type ComputeEncoder = Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>;
- type CopyEncoder = Retained<ProtocolObject<dyn MTLBlitCommandEncoder>>;
+ type CommandBuffer = MetalCommandBuffer;
+ type Buffer = Retained<ProtocolObject<dyn MTLBuffer>>;
type Event = Retained<ProtocolObject<dyn MTLEvent>>;
type Kernels = MetalKernels;
type Error = MetalError;
diff --git a/crates/uzu/src/backends/metal/native_buffer.rs b/crates/uzu/src/backends/metal/buffer.rs
similarity index 82%
rename from crates/uzu/src/backends/metal/native_buffer.rs
rename to crates/uzu/src/backends/metal/buffer.rs
index c36646fe5..84c0a4914 100644
--- a/crates/uzu/src/backends/metal/native_buffer.rs
+++ b/crates/uzu/src/backends/metal/buffer.rs
@@ -4,9 +4,9 @@ use metal::{MTLBuffer, MTLResourceExt};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::Metal;
-use crate::backends::common::NativeBuffer;
+use crate::backends::common::Buffer;
-impl NativeBuffer for Retained<ProtocolObject<dyn MTLBuffer>> {
+impl Buffer for Retained<ProtocolObject<dyn MTLBuffer>> {
type Backend = Metal;
fn set_label(
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index 8c7608cd8..8b72aaf00 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -1,12 +1,31 @@
-use std::{cell::Cell, ops::Deref};
+use std::cell::Cell;
use metal::{
- MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler, MTLCommandBufferStatus, MTLCommandEncoder, MTLEvent,
+ MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,
+ MTLCommandBufferHandler, MTLCommandBufferStatus, MTLCommandEncoder, MTLComputeCommandEncoder, MTLEvent,
};
-use objc2::{rc::Retained, runtime::ProtocolObject};
+use objc2::{Message, rc::Retained, runtime::ProtocolObject};
use super::Metal;
-use crate::backends::{common::CommandBuffer, metal::error::MetalError};
+use crate::backends::{
+ common::{
+ CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending,
+ },
+ metal::error::MetalError,
+};
+
+pub struct MetalCommandBuffer;
+
+impl CommandBuffer for MetalCommandBuffer {
+ type Backend = Metal;
+
+ type Initial = MetalCommandBufferInitial;
+ type Encoding = MetalCommandBufferEncoding;
+ type Executable = MetalCommandBufferExecutable;
+ type Pending = MetalCommandBufferPending;
+ type Completed = MetalCommandBufferCompleted;
+}
fn command_buffer_result(command_buffer: &ProtocolObject<dyn MTLCommandBuffer>) -> Result<(), MetalError> {
match (command_buffer.status(), command_buffer.error()) {
@@ -16,33 +35,110 @@ fn command_buffer_result(command_buffer: &ProtocolObject<dyn MTLCommandBuffer>)
}
}
-impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
- type Backend = Metal;
+pub struct MetalCommandBufferInitial {
+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+}
- fn with_compute_encoder<T>(
- &mut self,
- callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,
- ) -> T {
- let mut encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");
+impl MetalCommandBufferInitial {
+ pub fn new(command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>) -> Self {
+ Self {
+ command_buffer,
+ }
+ }
+}
+
+impl CommandBufferInitial for MetalCommandBufferInitial {
+ type CommandBuffer = MetalCommandBuffer;
+
+ fn start_encoding(self) -> MetalCommandBufferEncoding {
+ MetalCommandBufferEncoding {
+ command_buffer: self.command_buffer,
+ encoding_state: MetalCommandBufferEncodingEncodingState::None,
+ }
+ }
+}
+
+enum MetalCommandBufferEncodingEncodingState {
+ None,
+ Compute(Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>),
+ Blit(Retained<ProtocolObject<dyn MTLBlitCommandEncoder>>),
+}
+
+pub struct MetalCommandBufferEncoding {
+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+ encoding_state: MetalCommandBufferEncodingEncodingState,
+}
+
+impl MetalCommandBufferEncoding {
+ fn ensure_none(&mut self) {
+ match &mut self.encoding_state {
+ MetalCommandBufferEncodingEncodingState::None => return,
+ MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) => compute_encoder.end_encoding(),
+ MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) => blit_encoder.end_encoding(),
+ };
+
+ self.encoding_state = MetalCommandBufferEncodingEncodingState::None;
+ }
+
+ pub(super) fn ensure_compute(&mut self) -> &mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {
+ if !matches!(self.encoding_state, MetalCommandBufferEncodingEncodingState::Compute(_)) {
+ self.ensure_none();
+ self.encoding_state = MetalCommandBufferEncodingEncodingState::Compute(
+ self.command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder"),
+ );
+ }
+
+ let MetalCommandBufferEncodingEncodingState::Compute(compute_encoder) = &mut self.encoding_state else {
+ unreachable!()
+ };
+ compute_encoder
+ }
- let ret = callback(&mut encoder);
+ fn ensure_blit(&mut self) -> &mut Retained<ProtocolObject<dyn MTLBlitCommandEncoder>> {
+ if !matches!(self.encoding_state, MetalCommandBufferEncodingEncodingState::Blit(_)) {
+ self.ensure_none();
+ self.encoding_state = MetalCommandBufferEncodingEncodingState::Blit(
+ self.command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder"),
+ );
+ }
- encoder.end_encoding();
+ let MetalCommandBufferEncodingEncodingState::Blit(blit_encoder) = &mut self.encoding_state else {
+ unreachable!()
+ };
+ blit_encoder
+ }
+}
- ret
+impl Drop for MetalCommandBufferEncoding {
+ fn drop(&mut self) {
+ self.ensure_none();
}
+}
- fn with_copy_encoder<T>(
+impl CommandBufferEncoding for MetalCommandBufferEncoding {
+ type CommandBuffer = MetalCommandBuffer;
+
+ fn encode_copy(
&mut self,
- callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,
- ) -> T {
- let mut encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");
+ src: &Retained<ProtocolObject<dyn MTLBuffer>>,
+ dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,
+ size: usize,
+ ) {
+ assert!(src.length() >= size && dst.length() >= size);
- let ret = callback(&mut encoder);
+ self.ensure_blit().copy_buffer_to_buffer(src, 0, dst, 0, size);
+ }
- encoder.end_encoding();
+ fn encode_fill(
+ &mut self,
+ dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,
+ range: std::ops::Range<usize>,
+ value: u8,
+ ) {
+ assert!(range.end > range.start && range.end <= dst.length());
+ assert!(range.start % 4 == 0 && range.end % 4 == 0);
- ret
+ self.ensure_blit().fill_buffer_range_value(dst, range, value);
}
fn encode_wait_for_event(
@@ -50,7 +146,8 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
event: &Retained<ProtocolObject<dyn MTLEvent>>,
value: u64,
) {
- self.encode_wait_for_event_value(event, value);
+ self.ensure_none();
+ self.command_buffer.encode_wait_for_event_value(event, value);
}
fn encode_signal_event(
@@ -58,30 +155,75 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
event: &Retained<ProtocolObject<dyn MTLEvent>>,
value: u64,
) {
- self.encode_signal_event_value(event, value);
+ self.ensure_none();
+ self.command_buffer.encode_signal_event_value(event, value);
}
fn add_completion_handler(
&mut self,
- handler: impl FnOnce(Result<(), MetalError>) + 'static,
+ handler: impl FnOnce(Result<&MetalCommandBufferCompleted, MetalError>) + 'static,
) {
let cell = Cell::new(Some(handler));
- self.deref().add_completed_handler(&MTLCommandBufferHandler::new(move |command_buffer| {
- cell.take().expect("completion handler called more than once")(command_buffer_result(command_buffer))
+ self.command_buffer.add_completed_handler(&MTLCommandBufferHandler::new(move |command_buffer| {
+ let cbuf_ref = MetalCommandBufferCompleted {
+ command_buffer: command_buffer.retain(),
+ };
+ cell.take().expect("completion handler called more than once")(
+ command_buffer_result(command_buffer).map(|_| &cbuf_ref),
+ )
}));
}
- fn submit(&mut self) {
- self.commit();
+ fn end_encoding(mut self) -> <Self::CommandBuffer as CommandBuffer>::Executable {
+ self.ensure_none();
+
+ MetalCommandBufferExecutable {
+ command_buffer: self.command_buffer.clone(),
+ }
+ }
+}
+
+pub struct MetalCommandBufferExecutable {
+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+}
+
+impl CommandBufferExecutable for MetalCommandBufferExecutable {
+ type CommandBuffer = MetalCommandBuffer;
+
+ fn submit(self) -> MetalCommandBufferPending {
+ self.command_buffer.commit();
+
+ MetalCommandBufferPending {
+ command_buffer: self.command_buffer,
+ }
}
+}
+
+pub struct MetalCommandBufferPending {
+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+}
+
+impl CommandBufferPending for MetalCommandBufferPending {
+ type CommandBuffer = MetalCommandBuffer;
- fn wait_until_completed(&self) -> Result<(), MetalError> {
- self.deref().wait_until_completed();
- command_buffer_result(self)
+ fn wait_until_completed(self) -> Result<MetalCommandBufferCompleted, MetalError> {
+ self.command_buffer.wait_until_completed();
+
+ command_buffer_result(&self.command_buffer).map(|_| MetalCommandBufferCompleted {
+ command_buffer: self.command_buffer,
+ })
}
+}
+
+pub struct MetalCommandBufferCompleted {
+ command_buffer: Retained<ProtocolObject<dyn MTLCommandBuffer>>,
+}
+
+impl CommandBufferCompleted for MetalCommandBufferCompleted {
+ type CommandBuffer = MetalCommandBuffer;
fn gpu_execution_time_ms(&self) -> Option<f64> {
- match (self.kernel_start_time(), self.kernel_end_time()) {
+ match (self.command_buffer.kernel_start_time(), self.command_buffer.kernel_end_time()) {
(Some(start), Some(end)) => Some((end - start) * 1000.0),
_ => None,
}
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index 89973e2a0..69e10c1ed 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -1,14 +1,17 @@
use std::{cell::RefCell, collections::HashMap, env, rc::Rc};
use metal::{
- MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandBuffer, MTLCommandQueue,
- MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,
- MTLLibrary, MTLResourceOptions,
+ MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue, MTLCommandQueueExt,
+ MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues, MTLLibrary,
+ MTLResourceOptions,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{Metal, error::MetalError, kernel, metal_extensions::LibraryPipelineExtensions};
-use crate::backends::common::{Allocator, Context, DeviceClass as CommonDeviceClass};
+use crate::backends::{
+ common::{Context, DeviceClass as CommonDeviceClass},
+ metal::command_buffer::MetalCommandBufferInitial,
+};
/// Apple GPU architecture generation.
/// Based on Apple GPU family naming convention (e.g., "applegpu_g13p").
@@ -127,7 +130,6 @@ pub struct MetalContext {
architecture: DeviceArchitecture,
library: Retained<ProtocolObject<dyn MTLLibrary>>,
pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
- allocator: Allocator<Metal>,
}
impl MetalContext {
@@ -190,13 +192,12 @@ impl Context for MetalContext {
let architecture = DeviceArchitecture::from_device(&device);
- Ok(Rc::new_cyclic(|weak_self| Self {
+ Ok(Rc::new(Self {
device,
command_queue,
architecture,
library,
pipeline_cache: RefCell::new(HashMap::new()),
- allocator: Allocator::new(weak_self.clone()),
}))
}
@@ -221,10 +222,6 @@ impl Context for MetalContext {
matches!(upper.as_str(), "1" | "YES" | "TRUE")
}
- fn allocator(&self) -> &Allocator<Metal> {
- &self.allocator
- }
-
fn create_buffer(
&self,
size: usize,
@@ -232,8 +229,10 @@ impl Context for MetalContext {
self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).ok_or(MetalError::CannotCreateBuffer)
}
- fn create_command_buffer(&self) -> Result<Retained<ProtocolObject<dyn MTLCommandBuffer>>, MetalError> {
- self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)
+ fn create_command_buffer(&self) -> Result<MetalCommandBufferInitial, MetalError> {
+ Ok(MetalCommandBufferInitial::new(
+ self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,
+ ))
}
fn create_event(&self) -> Result<Retained<ProtocolObject<dyn MTLEvent>>, MetalError> {
diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rs
deleted file mode 100644
index e74776e15..000000000
--- a/crates/uzu/src/backends/metal/copy_encoder.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use std::ops::Range;
-
-use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::Metal;
-use crate::backends::common::{Backend, CopyEncoder};
-
-impl CopyEncoder for Retained<ProtocolObject<dyn MTLBlitCommandEncoder>> {
- type Backend = Metal;
-
- fn encode_copy(
- &mut self,
- src: &<Self::Backend as Backend>::NativeBuffer,
- dst: &mut <Self::Backend as Backend>::NativeBuffer,
- size: usize,
- ) {
- assert!(src.length() >= size && dst.length() >= size);
-
- self.copy_buffer_to_buffer(src, 0, dst, 0, size);
- }
-
- fn encode_fill(
- &mut self,
- dst: &mut <Self::Backend as Backend>::NativeBuffer,
- range: Range<usize>,
- value: u8,
- ) {
- assert!(range.end > range.start && range.end <= dst.length());
- assert!(range.start % 4 == 0 && range.end % 4 == 0);
-
- self.fill_buffer_range_value(dst, range, value);
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index d9482d193..f39fd382c 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -6,9 +6,12 @@ pub use dispatch_descriptor::choose_dispatch_descriptor;
use crate::{
DataType,
backends::{
- common::kernel::matmul::{
- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulArguments,
- MatmulError, MatmulKernel,
+ common::{
+ Backend, CommandBuffer,
+ kernel::matmul::{
+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,
+ MatmulArguments, MatmulError, MatmulKernel,
+ },
},
metal::{Metal, context::MetalContext},
},
@@ -29,7 +32,7 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {
fn encode(
&mut self,
context: &MetalContext,
- encoder: &mut <Metal as crate::backends::common::Backend>::ComputeEncoder,
+ encoder: &mut <<Metal as Backend>::CommandBuffer as CommandBuffer>::Encoding,
arguments: FullPrecisionMatmulArguments<Metal>,
) {
let mut matmul_arguments = MatmulArguments {
diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
deleted file mode 100644
index 9c6e5cf95..000000000
--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
+++ /dev/null
@@ -1,84 +0,0 @@
-use metal::{MTLBuffer, MTLCompareFunction, MTLComputeCommandEncoder};
-use objc2::{msg_send, rc::Retained, runtime::ProtocolObject};
-
-/// Low-level, unsafe conditional control of Metal encoders.
-/// This is internal; users should prefer the safe `ComputeEncoderConditional::condition`.
-trait ComputeEncoderRawConditional {
- fn encode_start_if(
- &self,
- predicate: &ProtocolObject<dyn MTLBuffer>,
- offset: usize,
- comparison: MTLCompareFunction,
- reference_value: u32,
- );
-
- fn encode_start_else(&self);
-
- fn encode_end_if(&self) -> bool;
-}
-
-/// Safe conditional wrapper that uses the raw unsafe methods internally.
-pub trait ComputeEncoderConditional {
- fn condition<IfBlock, ElseBlock>(
- &mut self,
- predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,
- offset: usize,
- if_block: IfBlock,
- else_block: Option<ElseBlock>,
- ) where
- IfBlock: FnOnce(&mut Self),
- ElseBlock: FnOnce(&mut Self);
-}
-
-impl ComputeEncoderRawConditional for Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {
- fn encode_start_if(
- &self,
- predicate: &ProtocolObject<dyn MTLBuffer>,
- offset: usize,
- comparison: MTLCompareFunction,
- reference_value: u32,
- ) {
- let _: () = unsafe {
- msg_send![
- self,
- encodeStartIf: predicate,
- offset: offset,
- comparison: comparison,
- referenceValue: reference_value
- ]
- };
- }
-
- fn encode_start_else(&self) {
- let _: () = unsafe { msg_send![self, encodeStartElse] };
- }
-
- fn encode_end_if(&self) -> bool {
- let result: bool = unsafe { msg_send![self, encodeEndIf] };
- result
- }
-}
-
-impl<T> ComputeEncoderConditional for T
-where
- T: ComputeEncoderRawConditional,
-{
- fn condition<IfBlock, ElseBlock>(
- &mut self,
- predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,
- offset: usize,
- if_block: IfBlock,
- else_block: Option<ElseBlock>,
- ) where
- IfBlock: FnOnce(&mut Self),
- ElseBlock: FnOnce(&mut Self),
- {
- self.encode_start_if(predicate, offset, MTLCompareFunction::Equal, 0);
- if_block(self);
- if let Some(else_block) = else_block {
- self.encode_start_else();
- else_block(self);
- }
- self.encode_end_if();
- }
-}
diff --git a/crates/uzu/src/backends/metal/metal_extensions/mod.rs b/crates/uzu/src/backends/metal/metal_extensions/mod.rs
index fd8b90ecb..963d89656 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/mod.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/mod.rs
@@ -1,10 +1,8 @@
-mod compute_command_encoder_extensions_conditional;
mod compute_command_encoder_extensions_set_value;
mod data_type;
mod function_constant_values_extensions_set_value;
mod library_extensions_pipeline;
-pub use compute_command_encoder_extensions_conditional::ComputeEncoderConditional;
pub use compute_command_encoder_extensions_set_value::ComputeEncoderSetValue;
pub use data_type::MetalDataTypeExt;
pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 64ed3aecb..2a7ef3c41 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -1,12 +1,11 @@
mod backend;
+mod buffer;
mod command_buffer;
mod context;
-mod copy_encoder;
mod error;
mod event;
mod kernel;
mod metal_extensions;
-mod native_buffer;
pub use backend::Metal;
pub use kernel::matmul::choose_dispatch_descriptor;
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 18dcfdf47..6c5bf3e02 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -9,7 +9,7 @@ use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
@@ -111,9 +111,7 @@ impl<B: Backend> Classifier<B> {
num_labels,
);
- self.context.reset_command_buffer();
-
- let encoding_params = EncodingParameters::new(false, true, false);
+ let encoding_params = EncodingParameters::new();
self.context
.embed
@@ -162,8 +160,12 @@ impl<B: Backend> Classifier<B> {
.encode(&mut state, &encoding_params, &mut self.context.command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context.command_buffer.submit();
- self.context.command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ self.context
+ .replace_command_buffer()
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
let logits = self.copy_logits_from_state(&state)?;
@@ -191,9 +193,7 @@ impl<B: Backend> Classifier<B> {
num_labels,
);
- self.context.reset_command_buffer();
-
- let encoding_params = EncodingParameters::new(false, true, false);
+ let encoding_params = EncodingParameters::new();
self.context
.embed
@@ -221,8 +221,12 @@ impl<B: Backend> Classifier<B> {
.encode(&mut state, &encoding_params, &mut self.context.command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context.command_buffer.submit();
- self.context.command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ self.context
+ .replace_command_buffer()
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
let logits = self.copy_logits_from_state(&state)?;
diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rs
index ef05c7ef3..4e73fa3c7 100644
--- a/crates/uzu/src/classifier/classifier_context.rs
+++ b/crates/uzu/src/classifier/classifier_context.rs
@@ -2,7 +2,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
use crate::{
DataType,
- backends::common::{Backend, Context, Kernels, kernel::SigmoidKernel},
+ backends::common::{Backend, CommandBuffer, CommandBufferInitial, Context, Kernels, kernel::SigmoidKernel},
classifier::ClassifierError,
config::{ClassifierModelConfig, ModelMetadata},
encodable_block::{
@@ -20,7 +20,7 @@ use crate::{
pub struct ClassifierContext<B: Backend> {
pub context: Rc<B::Context>,
- pub command_buffer: B::CommandBuffer,
+ pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
pub scratch_buffers: ScratchBuffers<B>,
@@ -45,7 +45,8 @@ impl<B: Backend> ClassifierContext<B> {
pub fn new(model_path: &Path) -> Result<Self, Error> {
let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
- let command_buffer = context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?;
+ let command_buffer =
+ context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?.start_encoding();
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -310,7 +311,10 @@ impl<B: Backend> ClassifierContext<B> {
Ok(Rc::new(rotation))
}
- pub fn reset_command_buffer(&mut self) {
- self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");
+ pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {
+ std::mem::replace(
+ &mut self.command_buffer,
+ self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),
+ )
}
}
diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rs
index f0fab5350..11caea45b 100644
--- a/crates/uzu/src/encodable_block/activation.rs
+++ b/crates/uzu/src/encodable_block/activation.rs
@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{ActivationKernel, Kernels},
},
config::Activation as ActivationConfig,
@@ -40,16 +40,12 @@ impl<B: Backend> Activation<B> {
}
impl<B: Backend> EncodableBlock<B> for Activation<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let input_array = arrays[0].borrow();
let output_array = arrays[1].borrow_mut();
@@ -70,7 +66,8 @@ impl<B: Backend> EncodableBlock<B> for Activation<B> {
output_array.buffer().borrow_mut().deref_mut(),
n as u32,
act_type,
- encoder,
+ command_buffer,
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index bf8f729fa..1a8bf6bf6 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -139,28 +139,9 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
-
- if parameters.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
- }
- Ok(())
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- compute_encoder: &mut B::ComputeEncoder,
- ) {
let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
let qkv_array = qkv_binding[0].borrow();
@@ -258,7 +239,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
// For classifiers (no KV cache): extract values from QKV into a dedicated extracted_values buffer.
if !has_kv_cache {
self.update_kv_cache_inplace_kernel.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
qkv_buf_borrow.deref(),
// keys already in desired layout; harmless overwrite
rotated_keys_buf_borrow.deref_mut(),
@@ -269,7 +250,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
suffix_length as u32,
0u32,
max_sequence_length as u32,
- compute_encoder,
+ command_buffer,
);
}
@@ -293,7 +274,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
};
let attention_bias_buf_rc = attention_bias_array_borrow.as_ref().map(|a| a.buffer());
let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());
- let attention_bias_buffer: Option<&B::NativeBuffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());
+ let attention_bias_buffer: Option<&B::Buffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());
let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);
let sums_binding = state.arrays(&[ArrayId::AttentionSums]);
@@ -314,7 +295,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
let sinks_borrow = sinks_binding.as_ref().map(|binding| binding[0].borrow());
let sinks_buf_rc = sinks_borrow.as_ref().map(|b| b.buffer());
let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());
- let sinks_buffer: Option<&B::NativeBuffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());
+ let sinks_buffer: Option<&B::Buffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());
// Only update KV cache for LLM mode (not for classifiers)
if has_kv_cache {
@@ -329,16 +310,16 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
suffix_length as u32,
segment_prefix_length as u32,
max_sequence_length as u32,
- compute_encoder,
+ command_buffer,
);
}
- let key_cache_buffer: &B::NativeBuffer = if has_kv_cache {
+ let key_cache_buffer: &B::Buffer = if has_kv_cache {
key_cache_buf_borrow.as_ref().unwrap().deref()
} else {
rotated_keys_buf_borrow.deref()
};
- let value_cache_buffer: &B::NativeBuffer = if has_kv_cache {
+ let value_cache_buffer: &B::Buffer = if has_kv_cache {
value_cache_buf_borrow.as_ref().unwrap().deref()
} else {
extracted_values_buf_borrow.as_ref().unwrap().deref()
@@ -383,7 +364,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
scale,
};
self.gemm_block
- .encode(state.context(), compute_encoder, args)
+ .encode(state.context(), command_buffer, args)
.expect("Failed to encode AttentionGemmBlock");
},
KernelVariant::SinglePass => {
@@ -410,7 +391,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
sinks_buffer,
num_heads as u32,
suffix_length as u32,
- compute_encoder,
+ command_buffer,
)
},
KernelVariant::TwoPass => {
@@ -443,7 +424,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
mask_q_seq_stride_opt,
mask_head_stride_opt,
sinks_buffer,
- compute_encoder,
+ command_buffer,
);
kernel_pass2.encode(
partials_buf_borrow.deref(),
@@ -452,10 +433,12 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
attention_output_buf_borrow.deref_mut(),
num_heads as u32,
suffix_length as u32,
- compute_encoder,
+ command_buffer,
);
},
}
+
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index 1c5637417..33b3ca455 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -2,11 +2,9 @@ use std::rc::Rc;
use objc2::rc::autoreleasepool;
-#[cfg(not(feature = "tracing"))]
-use crate::backends::common::CommandBuffer;
use crate::{
DataType,
- backends::common::Backend,
+ backends::common::{Backend, CommandBuffer},
config::TransformerLayerConfig,
encodable_block::{
Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,
@@ -249,18 +247,9 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- #[cfg(not(feature = "tracing"))]
- {
- if self.supports_shared_encoder() {
- command_buffer
- .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
- return Ok(());
- }
- }
-
#[cfg(feature = "tracing")]
let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
@@ -353,72 +342,4 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
let _ = parameters;
Ok(())
}
-
- fn supports_shared_encoder(&self) -> bool {
- #[cfg(feature = "tracing")]
- {
- false
- }
-
- #[cfg(not(feature = "tracing"))]
- {
- self.copy_main_to_shortcut_mixer.supports_shared_encoder()
- && self.pre_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
- && self.qkv_projection.supports_shared_encoder()
- && self.qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
- && self.rope.supports_shared_encoder()
- && self.attention.supports_shared_encoder()
- && self.out_projection.supports_shared_encoder()
- && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
- && self.mixer_residual_add.supports_shared_encoder()
- && self.copy_main_to_shortcut_mlp.supports_shared_encoder()
- && self.pre_mlp_norm.supports_shared_encoder()
- && self.mlp.supports_shared_encoder()
- && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
- && self.mlp_residual_add.supports_shared_encoder()
- }
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
- debug_assert!(
- self.supports_shared_encoder(),
- "encode_with_shared_encoder called on unsupported ClassifierLayer"
- );
-
- self.copy_main_to_shortcut_mixer.encode_with_shared_encoder(state, parameters, encoder);
-
- if let Some(ref pre_attn_norm) = self.pre_attention_norm {
- pre_attn_norm.encode_with_shared_encoder(state, parameters, encoder);
- }
-
- self.qkv_projection.encode_with_shared_encoder(state, parameters, encoder);
- if let Some(ref qk_norm) = self.qk_norm {
- qk_norm.encode_with_shared_encoder(state, parameters, encoder);
- }
- self.rope.encode_with_shared_encoder(state, parameters, encoder);
- self.attention.encode_with_shared_encoder(state, parameters, encoder);
- self.out_projection.encode_with_shared_encoder(state, parameters, encoder);
-
- if let Some(ref post_attn_norm) = self.post_attention_norm {
- post_attn_norm.encode_with_shared_encoder(state, parameters, encoder);
- }
-
- self.mixer_residual_add.encode_with_shared_encoder(state, parameters, encoder);
-
- self.copy_main_to_shortcut_mlp.encode_with_shared_encoder(state, parameters, encoder);
-
- self.pre_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);
- self.mlp.encode_with_shared_encoder(state, parameters, encoder);
-
- if let Some(ref post_mlp_norm) = self.post_mlp_norm {
- post_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);
- }
-
- self.mlp_residual_add.encode_with_shared_encoder(state, parameters, encoder);
- }
}
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index 56f88326f..75b4338c7 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -4,7 +4,7 @@ use std::rc::Rc;
use crate::{
DataType, DecoderConfig,
- backends::common::Backend,
+ backends::common::{Backend, CommandBuffer},
config::{DecoderLayerType, MixerConfig},
encodable_block::{
EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,
@@ -162,20 +162,11 @@ impl<B: Backend> Decoder<B> {
}
impl<B: Backend> EncodableBlock<B> for Decoder<B> {
- fn encode_with_shared_encoder(
- &self,
- _state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- _encoder: &mut B::ComputeEncoder,
- ) {
- unimplemented!("Decoder does not support shared encoder")
- }
-
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
self.embed.encode(state, parameters, command_buffer)?;
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
index 8b1e0a85a..c9e0fc151 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{FullPrecisionEmbeddingLookupKernel, Kernels},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -42,7 +42,7 @@ pub enum FullPrecisionEmbeddingLookupError<B: Backend> {
pub struct FullPrecisionEmbeddingLookup<B: Backend> {
kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
- weights_buffer: Rc<RefCell<B::NativeBuffer>>,
+ weights_buffer: Rc<RefCell<B::Buffer>>,
vocab_size: u32,
model_dim: u32,
input_scale: f32,
@@ -91,16 +91,12 @@ impl<B: Backend> FullPrecisionEmbeddingLookup<B> {
}
impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
let batch_size = state.active_suffix_length();
let token_ids_array = arrays[0].borrow_mut();
@@ -114,7 +110,8 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {
self.vocab_size,
self.model_dim,
self.input_scale,
- encoder,
+ command_buffer,
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
index 2af5bc31b..d993926f0 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
@@ -10,7 +10,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -42,7 +42,7 @@ pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {
pub struct FullPrecisionEmbeddingReadout<B: Backend> {
kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
- weights_buffer: Rc<RefCell<B::NativeBuffer>>,
+ weights_buffer: Rc<RefCell<B::Buffer>>,
vocab_size: usize,
model_dim: usize,
}
@@ -94,20 +94,16 @@ impl<B: Backend> FullPrecisionEmbeddingReadout<B> {
}
impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
let batch_size = state.sampling_length();
if batch_size == 0 {
- return;
+ return Ok(());
}
let sampling_start = state.sampling_start();
@@ -117,7 +113,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {
self.kernel.borrow_mut().encode(
state.context(),
- encoder,
+ command_buffer,
FullPrecisionMatmulArguments {
a: input_array.buffer().borrow().deref(),
a_offset: sampling_start * self.model_dim * element_size,
@@ -129,5 +125,6 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {
output_dim: self.vocab_size,
},
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rs
index 9c5daed29..bde5e27ff 100644
--- a/crates/uzu/src/encodable_block/full_precision_linear.rs
+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs
@@ -10,7 +10,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -50,8 +50,8 @@ pub enum FullPrecisionLinearError<B: Backend> {
pub struct FullPrecisionLinear<B: Backend> {
kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
- bias_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
- weights_buffer: Rc<RefCell<B::NativeBuffer>>,
+ bias_buffer: Option<Rc<RefCell<B::Buffer>>>,
+ weights_buffer: Rc<RefCell<B::Buffer>>,
input_dim: usize,
output_dim: usize,
input_array_id: ArrayId,
@@ -126,16 +126,12 @@ impl<B: Backend> FullPrecisionLinear<B> {
}
impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
let input_array = arrays[0].borrow_mut();
@@ -144,7 +140,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {
let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());
self.kernel.borrow_mut().encode(
state.context(),
- encoder,
+ command_buffer,
FullPrecisionMatmulArguments {
a: input_array.buffer().borrow().deref(),
a_offset: 0,
@@ -156,5 +152,6 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {
output_dim: self.output_dim,
},
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 25148726c..b51a5d85c 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -5,11 +5,9 @@ use std::rc::Rc;
use objc2::rc::autoreleasepool;
use super::MixerExecutables;
-#[cfg(not(feature = "tracing"))]
-use crate::backends::common::CommandBuffer;
use crate::{
DataType, DecoderLayerConfig,
- backends::common::Backend,
+ backends::common::{Backend, CommandBuffer},
config::{DecoderLayerType, MixerConfig},
encodable_block::{
Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,
@@ -260,20 +258,9 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- // In non-tracing builds, if every sub-block supports shared encoding,
- // we can run the entire layer in a single compute encoder.
- #[cfg(not(feature = "tracing"))]
- {
- if self.supports_shared_encoder() {
- command_buffer
- .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
- return Ok(());
- }
- }
-
#[cfg(feature = "tracing")]
let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
@@ -384,103 +371,4 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
let _ = parameters;
Ok(())
}
-
- fn supports_shared_encoder(&self) -> bool {
- #[cfg(feature = "tracing")]
- {
- false
- }
-
- #[cfg(not(feature = "tracing"))]
- {
- let mixer_ok = match &self.mixer {
- MixerExecutables::Attention {
- qkv_projection,
- qk_norm,
- rope,
- attention,
- out_projection,
- } => {
- qkv_projection.supports_shared_encoder()
- && qk_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
- && rope.supports_shared_encoder()
- && attention.supports_shared_encoder()
- && out_projection.supports_shared_encoder()
- },
- MixerExecutables::StateSpace {
- mixer,
- } => mixer.supports_shared_encoder(),
- MixerExecutables::ShortConv {
- mixer,
- } => mixer.supports_shared_encoder(),
- };
-
- self.copy_main_to_shortcut.supports_shared_encoder()
- && self.pre_attention_norm.supports_shared_encoder()
- && mixer_ok
- && self.post_attention_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
- && self.main_shortcut_add_swap.supports_shared_encoder()
- && self.pre_mlp_norm.supports_shared_encoder()
- && self.mlp.supports_shared_encoder()
- && self.post_mlp_norm.as_ref().map(|b| b.supports_shared_encoder()).unwrap_or(true)
- }
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
- debug_assert!(
- self.supports_shared_encoder(),
- "encode_with_shared_encoder called on unsupported LayerExecutables"
- );
-
- self.copy_main_to_shortcut.encode_with_shared_encoder(state, parameters, encoder);
- self.pre_attention_norm.encode_with_shared_encoder(state, parameters, encoder);
-
- match &self.mixer {
- MixerExecutables::Attention {
- qkv_projection,
- qk_norm,
- rope,
- attention,
- out_projection,
- } => {
- qkv_projection.encode_with_shared_encoder(state, parameters, encoder);
- if let Some(norm) = qk_norm {
- norm.encode_with_shared_encoder(state, parameters, encoder);
- }
- rope.encode_with_shared_encoder(state, parameters, encoder);
- attention.encode_with_shared_encoder(state, parameters, encoder);
- out_projection.encode_with_shared_encoder(state, parameters, encoder);
- },
- MixerExecutables::StateSpace {
- mixer,
- } => {
- mixer.encode_with_shared_encoder(state, parameters, encoder);
- },
- MixerExecutables::ShortConv {
- mixer,
- } => {
- mixer.encode_with_shared_encoder(state, parameters, encoder);
- },
- }
-
- if let Some(post_attention_norm) = &self.post_attention_norm {
- post_attention_norm.encode_with_shared_encoder(state, parameters, encoder);
- }
-
- self.main_shortcut_add_swap.encode_with_shared_encoder(state, parameters, encoder);
-
- self.pre_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);
- self.mlp.encode_with_shared_encoder(state, parameters, encoder);
-
- if let Some(post_mlp_norm) = &self.post_mlp_norm {
- post_mlp_norm.encode_with_shared_encoder(state, parameters, encoder);
- }
-
- self.main_shortcut_add_swap.encode_with_shared_encoder(state, parameters, encoder);
- }
}
diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rs
index a622ffe73..0c2ec7e55 100644
--- a/crates/uzu/src/encodable_block/layer_norm.rs
+++ b/crates/uzu/src/encodable_block/layer_norm.rs
@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{Kernels, LayerNormKernel},
},
config::{NormalizationConfig, UpcastMode},
@@ -33,7 +33,7 @@ pub struct LayerNorm<B: Backend> {
config: NormalizationConfig,
input_array_id: ArrayId,
output_array_id: ArrayId,
- scales_buffer: Rc<RefCell<B::NativeBuffer>>,
+ scales_buffer: Rc<RefCell<B::Buffer>>,
}
impl<B: Backend> LayerNorm<B> {
@@ -71,16 +71,12 @@ impl<B: Backend> LayerNorm<B> {
}
impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
@@ -111,7 +107,8 @@ impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
full_layer,
- encoder,
+ command_buffer,
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index 318fd7f28..75948b15e 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -139,34 +139,10 @@ impl<B: Backend> MambaMixer<B> {
}
impl<B: Backend> MambaMixer<B> {
- fn encode_pipeline_with_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- encoder: &mut B::ComputeEncoder,
- parameters: &EncodingParameters<B>,
- ) {
- let active_suffix_length = state.active_suffix_length();
- if active_suffix_length == 0 {
- return;
- }
-
- self.in_projection.encode_with_shared_encoder(state, parameters, encoder);
- self.run_split_inproj(state, encoder, active_suffix_length);
- self.run_conv_scan(state, encoder, active_suffix_length);
-
- if active_suffix_length == 1 {
- self.run_decode_ssm(state, encoder, active_suffix_length);
- } else {
- self.run_prefill_ssm(state, encoder, active_suffix_length);
- }
-
- self.out_projection.encode_with_shared_encoder(state, parameters, encoder);
- }
-
fn run_split_inproj(
&self,
state: &mut ForwardPassState<B>,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -199,14 +175,14 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
inner_dim as u32,
num_heads as u32,
- encoder,
+ command_buffer,
)
}
fn run_conv_scan(
&self,
state: &mut ForwardPassState<B>,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -238,7 +214,7 @@ impl<B: Backend> MambaMixer<B> {
conv_inputs.buffer().borrow().deref(),
weight_buf_borrow.deref(),
bias_buf_borrow.as_deref(),
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
x_arr.buffer().borrow_mut().deref_mut(),
b_arr.buffer().borrow_mut().deref_mut(),
c_arr.buffer().borrow_mut().deref_mut(),
@@ -250,7 +226,7 @@ impl<B: Backend> MambaMixer<B> {
suffix_length as u32,
inner_dim as u32,
proj_dim as u32,
- encoder,
+ command_buffer,
);
}
} else {
@@ -266,7 +242,7 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
suffix_length as u32,
conv_dim as u32,
- encoder,
+ command_buffer,
);
Some(buffer)
@@ -278,7 +254,7 @@ impl<B: Backend> MambaMixer<B> {
let padded_borrow = padded_buf.as_ref().map(|b| b.borrow());
let conv_inputs_borrow = conv_inputs.buffer();
let conv_inputs_ref = conv_inputs_borrow.borrow();
- let conv_source: &B::NativeBuffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());
+ let conv_source: &B::Buffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());
self.conv_scan.encode(
conv_source,
weight_buf_borrow.deref(),
@@ -294,7 +270,7 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
inner_dim as u32,
proj_dim as u32,
- encoder,
+ command_buffer,
)
}
}
@@ -303,7 +279,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_prefill_ssm(
&self,
state: &mut ForwardPassState<B>,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
suffix_length: usize,
) {
let base_arrays = state.arrays(&[
@@ -327,7 +303,7 @@ impl<B: Backend> MambaMixer<B> {
let skip_rc = self.skip_connection_weight.buffer();
let skip_borrow = skip_rc.borrow();
self.ssd_prefill.encode(
- encoder,
+ command_buffer,
SSDPrefillArguments {
x: x.buffer().borrow().deref(),
dt: dt.buffer().borrow().deref(),
@@ -354,7 +330,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_decode_ssm(
&self,
state: &mut ForwardPassState<B>,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -396,7 +372,7 @@ impl<B: Backend> MambaMixer<B> {
c.buffer().borrow().deref(),
skip_borrow.deref(),
z.buffer().borrow().deref(),
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
y.buffer().borrow_mut().deref_mut(),
state_arr.buffer().borrow_mut().deref_mut(),
group_size as u32,
@@ -408,7 +384,7 @@ impl<B: Backend> MambaMixer<B> {
suffix_length as u32,
h,
dh,
- encoder,
+ command_buffer,
);
}
@@ -438,56 +414,23 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- if self.supports_shared_encoder() {
- command_buffer
- .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));
-
- if parameters.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
- }
- return Ok(());
- }
-
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
return Ok(());
}
self.in_projection.encode(state, parameters, command_buffer)?;
-
- command_buffer.with_compute_encoder(|encoder| {
- self.run_split_inproj(state, encoder, active_suffix_length);
- self.run_conv_scan(state, encoder, active_suffix_length);
- if active_suffix_length == 1 {
- self.run_decode_ssm(state, encoder, active_suffix_length);
- } else {
- self.run_prefill_ssm(state, encoder, active_suffix_length);
- }
- });
-
- self.out_projection.encode(state, parameters, command_buffer)?;
-
- if parameters.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
+ self.run_split_inproj(state, command_buffer, active_suffix_length);
+ self.run_conv_scan(state, command_buffer, active_suffix_length);
+ if active_suffix_length == 1 {
+ self.run_decode_ssm(state, command_buffer, active_suffix_length);
+ } else {
+ self.run_prefill_ssm(state, command_buffer, active_suffix_length);
}
+ self.out_projection.encode(state, parameters, command_buffer)?;
Ok(())
}
-
- fn supports_shared_encoder(&self) -> bool {
- self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
- self.encode_pipeline_with_encoder(state, encoder, parameters);
- }
}
diff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rs
index df0d31348..36c41f65b 100644
--- a/crates/uzu/src/encodable_block/mlp.rs
+++ b/crates/uzu/src/encodable_block/mlp.rs
@@ -32,54 +32,11 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- params: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ params: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- if self.supports_shared_encoder() {
- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, params, encoder));
- } else {
- // Up
- self.up.encode(state, params, command_buffer)?;
-
- // Gate act+mul (fused_up -> hidden)
- command_buffer.with_compute_encoder(|encoder| {
- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
- let fused = arrays[0].borrow_mut();
- let hidden = arrays[1].borrow_mut();
- let m = fused.shape()[0] as i32;
- let fused_buf_rc = fused.buffer();
- let fused_buf_borrow = fused_buf_rc.borrow();
- let hidden_buf_rc = hidden.buffer();
- let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();
-
- self.gate
- .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)
- .expect("Failed to encode MLP activation/mul kernel");
- });
-
- // Down
- self.down.encode(state, params, command_buffer)?;
- }
-
- if params.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
- }
- Ok(())
- }
-
- fn supports_shared_encoder(&self) -> bool {
- self.up.supports_shared_encoder() && self.down.supports_shared_encoder()
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- params: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
// Up
- self.up.encode_with_shared_encoder(state, params, encoder);
+ self.up.encode(state, params, command_buffer)?;
// Gate act+mul (fused_up -> hidden)
let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
@@ -91,7 +48,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
let hidden_buf_rc = hidden.buffer();
let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();
self.gate
- .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)
+ .encode(command_buffer, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)
.expect("Failed to encode MLP activation/mul kernel");
drop(hidden_buf_borrow);
drop(fused_buf_borrow);
@@ -99,6 +56,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
drop(hidden);
// Down
- self.down.encode_with_shared_encoder(state, params, encoder);
+ self.down.encode(state, params, command_buffer)?;
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 27847edb9..a973d83c3 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -58,26 +58,14 @@ pub use tensor_copy::TensorCopy;
pub use transformer_layer::{embed_block, linear_block};
#[derive(Clone)]
-pub struct EncodingParameters<'a, B: Backend> {
- pub warmup: bool,
- pub enable_commit: bool,
- pub wait_until_completed: bool,
+pub struct EncodingParameters {
pub projection_step: Option<usize>,
- pub predicate: Option<&'a B::NativeBuffer>,
}
-impl<'a, B: Backend> EncodingParameters<'a, B> {
- pub fn new(
- warmup: bool,
- enable_commit: bool,
- wait_until_completed: bool,
- ) -> Self {
+impl EncodingParameters {
+ pub fn new() -> Self {
Self {
- warmup,
- enable_commit,
- wait_until_completed,
projection_step: None,
- predicate: None,
}
}
@@ -88,39 +76,13 @@ impl<'a, B: Backend> EncodingParameters<'a, B> {
self.projection_step = Some(projection_step);
self
}
-
- pub fn with_predicate(
- mut self,
- predicate: &'a B::NativeBuffer,
- ) -> Self {
- self.predicate = Some(predicate);
- self
- }
-
- pub fn predicate_ref(&self) -> Option<&B::NativeBuffer> {
- self.predicate
- }
}
pub trait EncodableBlock<B: Backend> {
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- );
-
- fn supports_shared_encoder(&self) -> bool {
- false
- }
-
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
- ) -> Result<(), B::Error> {
- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
- Ok(())
- }
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error>;
}
diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rs
index 07869e271..a5be081ba 100644
--- a/crates/uzu/src/encodable_block/moe_block.rs
+++ b/crates/uzu/src/encodable_block/moe_block.rs
@@ -12,7 +12,7 @@ use crate::{
Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,
array::Array,
backends::common::{
- Backend, CommandBuffer, CopyEncoder, Kernels,
+ Backend, CommandBuffer, CommandBufferEncoding, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
@@ -28,16 +28,16 @@ use crate::{
};
struct RouterBlock<B: Backend> {
- weights_buf: Rc<RefCell<B::NativeBuffer>>,
- biases_buf: Rc<RefCell<B::NativeBuffer>>,
+ weights_buf: Rc<RefCell<B::Buffer>>,
+ biases_buf: Rc<RefCell<B::Buffer>>,
}
#[derive(Clone)]
struct SharedMoeWeights<B: Backend> {
- pub w13_buf: Rc<RefCell<B::NativeBuffer>>,
- pub w2_buf: Rc<RefCell<B::NativeBuffer>>,
- pub up_biases_buf: Rc<RefCell<B::NativeBuffer>>,
- pub down_biases_buf: Rc<RefCell<B::NativeBuffer>>,
+ pub w13_buf: Rc<RefCell<B::Buffer>>,
+ pub w2_buf: Rc<RefCell<B::Buffer>>,
+ pub up_biases_buf: Rc<RefCell<B::Buffer>>,
+ pub down_biases_buf: Rc<RefCell<B::Buffer>>,
}
pub struct MoeBlock<B: Backend> {
@@ -192,8 +192,8 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let suffix_length = state.active_suffix_length();
let arrays = state.arrays(&[
@@ -219,7 +219,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
ArrayId::MoeTwoPassRowExpertMap,
]);
- let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<RefCell<B::NativeBuffer>> { array.borrow().buffer() };
+ let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<RefCell<B::Buffer>> { array.borrow().buffer() };
let mut array_iter = arrays.iter();
let main_buf = clone_buffer(array_iter.next().unwrap());
@@ -249,39 +249,37 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
let k_tile = 128;
// Clear internal MoE buffers
- command_buffer.with_copy_encoder(|encoder| {
- if suffix_length > 0 && k > 0 {
- let entries = suffix_length * k;
- let topk_bytes = entries * std::mem::size_of::<u32>();
- let tok2row_bytes = entries * std::mem::size_of::<i32>();
-
- // Clear topk_ids and tok2row buffers
- if topk_bytes > 0 {
- encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);
- }
- if tok2row_bytes > 0 {
- encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);
- }
+ if suffix_length > 0 && k > 0 {
+ let entries = suffix_length * k;
+ let topk_bytes = entries * std::mem::size_of::<u32>();
+ let tok2row_bytes = entries * std::mem::size_of::<i32>();
+
+ // Clear topk_ids and tok2row buffers
+ if topk_bytes > 0 {
+ command_buffer.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);
+ }
+ if tok2row_bytes > 0 {
+ command_buffer.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);
+ }
- // Clear hidden buffer
- let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
- if hidden_bytes > 0 {
- encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);
- }
+ // Clear hidden buffer
+ let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
+ if hidden_bytes > 0 {
+ command_buffer.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);
+ }
- // Clear y_partial buffer
- let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
- if y_partial_bytes > 0 {
- encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);
- }
+ // Clear y_partial buffer
+ let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
+ if y_partial_bytes > 0 {
+ command_buffer.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);
+ }
- // Clear x_perm buffer
- let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
- if x_perm_bytes > 0 {
- encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);
- }
+ // Clear x_perm buffer
+ let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
+ if x_perm_bytes > 0 {
+ command_buffer.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);
}
- });
+ }
// Use fused Router+TopK kernel for non-quantized routers
if self.model_dim % 4 != 0 {
@@ -295,69 +293,61 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
}
if suffix_length > 0 && e > 0 && k > 0 {
- command_buffer.with_compute_encoder(|encoder| {
- // Use the fused router+topk kernel
- self.router_topk_kernel.encode(
- main_buf.borrow().deref(),
- self.router.weights_buf.borrow().deref(),
- self.router.biases_buf.borrow().deref(),
- topk_ids_buf.borrow_mut().deref_mut(),
- topk_probs_buf.borrow_mut().deref_mut(),
- suffix_length as u32,
- self.model_dim as u32,
- e as u32,
- k as u32,
- self.router_renorm,
- encoder,
- );
- });
- }
-
- command_buffer.with_compute_encoder(|encoder| {
- self.counts_offsets_kernel.encode(
- topk_ids_buf.borrow().deref(),
- offsets_buf.borrow_mut().deref_mut(),
- sumk_buf.borrow_mut().deref_mut(),
- partials_buf.borrow_mut().deref_mut(),
+ // Use the fused router+topk kernel
+ self.router_topk_kernel.encode(
+ main_buf.borrow().deref(),
+ self.router.weights_buf.borrow().deref(),
+ self.router.biases_buf.borrow().deref(),
+ topk_ids_buf.borrow_mut().deref_mut(),
+ topk_probs_buf.borrow_mut().deref_mut(),
suffix_length as u32,
+ self.model_dim as u32,
e as u32,
k as u32,
- encoder,
+ self.router_renorm,
+ command_buffer,
);
- });
+ }
+
+ self.counts_offsets_kernel.encode(
+ topk_ids_buf.borrow().deref(),
+ offsets_buf.borrow_mut().deref_mut(),
+ sumk_buf.borrow_mut().deref_mut(),
+ partials_buf.borrow_mut().deref_mut(),
+ suffix_length as u32,
+ e as u32,
+ k as u32,
+ command_buffer,
+ );
let num_blocks = ((suffix_length + 255) / 256).max(1);
let num_tiles = ((e + 512 - 1) / 512).max(1);
- command_buffer.with_compute_encoder(|encoder| {
- self.scatter_bases_kernel.encode(
- partials_buf.borrow().deref(),
- block_bases_buf.borrow_mut().deref_mut(),
- block_alloc_buf.borrow_mut().deref_mut(),
- e as u32,
- num_blocks as u32,
- num_tiles as u32,
- 0u32,
- encoder,
- );
- });
- command_buffer.with_compute_encoder(|encoder| {
- self.scatter_map_kernel.encode(
- topk_ids_buf.borrow().deref(),
- topk_probs_buf.borrow().deref(),
- offsets_buf.borrow().deref(),
- block_bases_buf.borrow().deref(),
- block_alloc_buf.borrow().deref(),
- bucketed_ids_buf.borrow_mut().deref_mut(),
- bucketed_probs_buf.borrow_mut().deref_mut(),
- suffix_length as u32,
- e as u32,
- k as u32,
- num_blocks as u32,
- num_tiles as u32,
- tok2row_buf.borrow_mut().deref_mut(),
- encoder,
- );
- });
+ self.scatter_bases_kernel.encode(
+ partials_buf.borrow().deref(),
+ block_bases_buf.borrow_mut().deref_mut(),
+ block_alloc_buf.borrow_mut().deref_mut(),
+ e as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ 0u32,
+ command_buffer,
+ );
+ self.scatter_map_kernel.encode(
+ topk_ids_buf.borrow().deref(),
+ topk_probs_buf.borrow().deref(),
+ offsets_buf.borrow().deref(),
+ block_bases_buf.borrow().deref(),
+ block_alloc_buf.borrow().deref(),
+ bucketed_ids_buf.borrow_mut().deref_mut(),
+ bucketed_probs_buf.borrow_mut().deref_mut(),
+ suffix_length as u32,
+ e as u32,
+ k as u32,
+ num_blocks as u32,
+ num_tiles as u32,
+ tok2row_buf.borrow_mut().deref_mut(),
+ command_buffer,
+ );
self.gather_kernels.encode(
command_buffer,
@@ -467,36 +457,17 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
self.experts_two_pass_prefill_kernel.encode(command_buffer, args);
}
- command_buffer.with_compute_encoder(|encoder| {
- self.finalize_kernel.encode(
- tok2row_buf.borrow().deref(),
- topk_probs_buf.borrow().deref(),
- y_partial_buf.borrow().deref(),
- main_buf.borrow_mut().deref_mut(),
- suffix_length as u32,
- self.model_dim as u32,
- k as u32,
- encoder,
- );
- });
+ self.finalize_kernel.encode(
+ tok2row_buf.borrow().deref(),
+ topk_probs_buf.borrow().deref(),
+ y_partial_buf.borrow().deref(),
+ main_buf.borrow_mut().deref_mut(),
+ suffix_length as u32,
+ self.model_dim as u32,
+ k as u32,
+ command_buffer,
+ );
- if parameters.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
- }
Ok(())
}
-
- fn supports_shared_encoder(&self) -> bool {
- false
- }
-
- fn encode_with_shared_encoder(
- &self,
- _state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- _encoder: &mut B::ComputeEncoder,
- ) {
- unreachable!("MoeBlock does not support shared compute encoder");
- }
}
diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rs
index 0553ecef4..308b3fab6 100644
--- a/crates/uzu/src/encodable_block/normalization.rs
+++ b/crates/uzu/src/encodable_block/normalization.rs
@@ -5,7 +5,7 @@ use thiserror::Error;
use super::{EncodableBlock, EncodingParameters, LayerNorm, LayerNormError, RMSNorm, RMSNormError};
use crate::{
DataType,
- backends::common::Backend,
+ backends::common::{Backend, CommandBuffer},
config::NormalizationConfig,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -59,22 +59,15 @@ impl<B: Backend> Normalization<B> {
}
impl<B: Backend> EncodableBlock<B> for Normalization<B> {
- fn supports_shared_encoder(&self) -> bool {
- match self {
- Self::LayerNorm(layer_norm) => layer_norm.supports_shared_encoder(),
- Self::RMSNorm(rms_norm) => rms_norm.supports_shared_encoder(),
- }
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
match self {
- Self::LayerNorm(layer_norm) => layer_norm.encode_with_shared_encoder(state, parameters, encoder),
- Self::RMSNorm(rms_norm) => rms_norm.encode_with_shared_encoder(state, parameters, encoder),
+ Self::LayerNorm(layer_norm) => layer_norm.encode(state, parameters, command_buffer),
+ Self::RMSNorm(rms_norm) => rms_norm.encode(state, parameters, command_buffer),
}
}
}
diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rs
index 00caf4324..bf8010fac 100644
--- a/crates/uzu/src/encodable_block/pooling.rs
+++ b/crates/uzu/src/encodable_block/pooling.rs
@@ -21,16 +21,16 @@ enum PoolingKernel<B: Backend> {
impl<B: Backend> PoolingKernel<B> {
fn encode(
&self,
- input: &B::NativeBuffer,
- output: &mut B::NativeBuffer,
+ input: &B::Buffer,
+ output: &mut B::Buffer,
seq_len: u32,
hidden_dim: u32,
batch_size: u32,
- encoder: &mut B::ComputeEncoder,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) {
match self {
- Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
- Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
+ Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),
+ Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),
}
}
}
@@ -64,29 +64,9 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
-
- #[cfg(feature = "tracing")]
- {
- let output_pooling_trace = state.traces().borrow().output_pooling().clone();
- state.encode_copy_array(command_buffer, ArrayId::ClassifierPooling, output_pooling_trace);
- }
- Ok(())
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
let batch_size = 1;
let seq_len = state.aux_buffers_suffix_length();
@@ -100,7 +80,14 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {
seq_len as u32,
self.model_dim as u32,
batch_size,
- encoder,
+ command_buffer,
);
+
+ #[cfg(feature = "tracing")]
+ {
+ let output_pooling_trace = state.traces().borrow().output_pooling().clone();
+ state.encode_copy_array(command_buffer, ArrayId::ClassifierPooling, output_pooling_trace);
+ }
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rs
index c41133e6c..46ec73491 100644
--- a/crates/uzu/src/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/encodable_block/prediction_head.rs
@@ -39,17 +39,13 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- if self.supports_shared_encoder() {
- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
- } else {
- self.dense.encode(state, parameters, command_buffer)?;
- self.activation.encode(state, parameters, command_buffer)?;
- self.norm.encode(state, parameters, command_buffer)?;
- self.readout.encode(state, parameters, command_buffer)?;
- }
+ self.dense.encode(state, parameters, command_buffer)?;
+ self.activation.encode(state, parameters, command_buffer)?;
+ self.norm.encode(state, parameters, command_buffer)?;
+ self.readout.encode(state, parameters, command_buffer)?;
#[cfg(feature = "tracing")]
{
@@ -61,29 +57,6 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {
);
}
- if parameters.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
- }
Ok(())
}
-
- fn supports_shared_encoder(&self) -> bool {
- self.dense.supports_shared_encoder()
- && self.activation.supports_shared_encoder()
- && self.norm.supports_shared_encoder()
- && self.readout.supports_shared_encoder()
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
- self.dense.encode_with_shared_encoder(state, parameters, encoder);
- self.activation.encode_with_shared_encoder(state, parameters, encoder);
- self.norm.encode_with_shared_encoder(state, parameters, encoder);
- self.readout.encode_with_shared_encoder(state, parameters, encoder);
- }
}
diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
index 4c8bb857a..0c25f43bf 100644
--- a/crates/uzu/src/encodable_block/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{Kernels, QKNormKernel},
},
config::{NormalizationConfig, UpcastMode},
@@ -34,8 +34,8 @@ pub struct QKNorm<B: Backend> {
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
qkv_array_id: ArrayId,
- query_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
- key_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
+ query_scales_buffer: Option<Rc<RefCell<B::Buffer>>>,
+ key_scales_buffer: Option<Rc<RefCell<B::Buffer>>>,
num_q_heads: usize,
num_kv_heads: usize,
head_dim: usize,
@@ -124,16 +124,12 @@ impl<B: Backend> QKNorm<B> {
}
impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
let qkv_shape = {
let qkv_array = qkv_binding[0].borrow();
@@ -148,7 +144,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
(&self.query_kernel, &self.query_scales_buffer, &self.query_config)
{
query_kernel.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
query_scales_buffer.borrow().deref(),
qkv_array.buffer().borrow_mut().deref_mut(),
batch_size,
@@ -160,7 +156,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
0,
self.num_q_heads as u32,
query_config.upcast_mode == UpcastMode::FullLayer,
- encoder,
+ command_buffer,
);
}
@@ -169,7 +165,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
(&self.key_kernel, &self.key_scales_buffer, &self.key_config)
{
key_kernel.encode(
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
key_scales_buffer.borrow().deref(),
qkv_array.buffer().borrow_mut().deref_mut(),
batch_size,
@@ -181,8 +177,9 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
self.num_q_heads as u32,
self.num_kv_heads as u32,
key_config.upcast_mode == UpcastMode::FullLayer,
- encoder,
+ command_buffer,
);
}
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
index ff9494f18..ac50416da 100644
--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
+++ b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend, Context, NativeBuffer,
+ Backend, Buffer, CommandBuffer, Context,
kernel::{Kernels, QuantizedEmbeddingLookupKernel},
},
config::QuantizationMode,
@@ -71,9 +71,9 @@ pub enum QuantizedEmbeddingLookupError<B: Backend> {
pub struct QuantizedEmbeddingLookup<B: Backend> {
kernel: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
- weights_buffer: Rc<RefCell<B::NativeBuffer>>,
- scales_buffer: Rc<RefCell<B::NativeBuffer>>,
- biases_buffer: Rc<RefCell<B::NativeBuffer>>,
+ weights_buffer: Rc<RefCell<B::Buffer>>,
+ scales_buffer: Rc<RefCell<B::Buffer>>,
+ biases_buffer: Rc<RefCell<B::Buffer>>,
mode: QuantizationMode,
input_scale: f32,
vocab_size: u32,
@@ -239,16 +239,12 @@ impl<B: Backend> QuantizedEmbeddingLookup<B> {
}
impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
let batch_size = state.active_suffix_length();
let token_ids_array = arrays[0].borrow_mut();
@@ -272,7 +268,8 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {
self.group_size,
self.input_scale,
quant_mode,
- encoder,
+ command_buffer,
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
index 7abaa606f..791237300 100644
--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend, Context, NativeBuffer,
+ Backend, Buffer, CommandBuffer, Context,
kernel::quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
QuantizedMatmulKernelEncodable, QuantizedMatmulType,
@@ -71,9 +71,9 @@ pub enum QuantizedEmbeddingReadoutError<B: Backend> {
pub struct QuantizedEmbeddingReadout<B: Backend> {
kernel: QuantizedMatmulKernelEncodable<B>,
- weights_buffer: Rc<RefCell<B::NativeBuffer>>,
- scales_buffer: Rc<RefCell<B::NativeBuffer>>,
- biases_buffer: Rc<RefCell<B::NativeBuffer>>,
+ weights_buffer: Rc<RefCell<B::Buffer>>,
+ scales_buffer: Rc<RefCell<B::Buffer>>,
+ biases_buffer: Rc<RefCell<B::Buffer>>,
vocab_size: usize,
model_dim: usize,
}
@@ -229,20 +229,16 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {
}
impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
let batch_size = state.sampling_length();
if batch_size == 0 {
- return;
+ return Ok(());
}
let sampling_start = state.sampling_start();
@@ -254,7 +250,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {
self.kernel
.encode(
- encoder,
+ command_buffer,
QuantizedMatmulArguments {
a_buffer: input_array.buffer().borrow().deref(),
a_offset,
@@ -269,5 +265,6 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {
},
)
.expect("Failed to encode quantized embedding readout kernel");
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rs
index 173d53320..c2fce2e9c 100644
--- a/crates/uzu/src/encodable_block/quantized_linear.rs
+++ b/crates/uzu/src/encodable_block/quantized_linear.rs
@@ -10,7 +10,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{
Kernels, TensorAddBiasKernel,
quant_matmul::{
@@ -88,10 +88,10 @@ pub enum QuantizedLinearError<B: Backend> {
pub struct QuantizedLinear<B: Backend> {
kernel: QuantizedMatmulKernelEncodable<B>,
bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
- biases_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
- weights_buffer: Rc<RefCell<B::NativeBuffer>>,
- scales_buffer: Rc<RefCell<B::NativeBuffer>>,
- zero_points_or_biases_buffer: Rc<RefCell<B::NativeBuffer>>,
+ biases_buffer: Option<Rc<RefCell<B::Buffer>>>,
+ weights_buffer: Rc<RefCell<B::Buffer>>,
+ scales_buffer: Rc<RefCell<B::Buffer>>,
+ zero_points_or_biases_buffer: Rc<RefCell<B::Buffer>>,
quantization_type: QuantizedMatmulType,
input_dim: usize,
output_dim: usize,
@@ -245,16 +245,12 @@ impl<B: Backend> QuantizedLinear<B> {
}
impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
let input_array = arrays[0].borrow_mut();
@@ -264,7 +260,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {
self.kernel
.encode(
- encoder,
+ command_buffer,
QuantizedMatmulArguments {
a_buffer: input_array.buffer().borrow().deref(),
a_offset: 0,
@@ -282,15 +278,15 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {
if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {
let total_length = batch_size * self.output_dim;
- bias_add_kernel.encode_if(
- None::<&B::NativeBuffer>,
+ bias_add_kernel.encode(
+ None::<&B::Buffer>,
biases_buffer.borrow().deref(),
output_buf_borrow.deref_mut(),
self.output_dim as u32,
total_length as u32,
- encoder,
- parameters.predicate_ref(),
+ command_buffer,
);
}
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 699787612..13a1c7483 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -12,7 +12,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{Kernels, RMSNormKernel},
},
config::{NormalizationConfig, UpcastMode},
@@ -33,7 +33,7 @@ pub struct RMSNorm<B: Backend> {
config: NormalizationConfig,
input_array_id: ArrayId,
output_array_id: ArrayId,
- scales_buffer: Rc<RefCell<B::NativeBuffer>>,
+ scales_buffer: Rc<RefCell<B::Buffer>>,
use_sampling_range: bool,
}
@@ -85,16 +85,12 @@ impl<B: Backend> RMSNorm<B> {
}
impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
@@ -117,7 +113,7 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {
};
let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));
if batch_len == 0 {
- return;
+ return Ok(());
}
let row_size_in_bytes = input_shape[1] * input_elem_size;
@@ -138,7 +134,8 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
self.config.upcast_mode == UpcastMode::FullLayer,
- encoder,
+ command_buffer,
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rs
index 6660fbe67..66ca919f2 100644
--- a/crates/uzu/src/encodable_block/rope.rs
+++ b/crates/uzu/src/encodable_block/rope.rs
@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{Kernels, RopeKernel},
},
forward_pass::state::{ArrayId, ForwardPassState, RopeType},
@@ -31,16 +31,12 @@ impl<B: Backend> Rope<B> {
}
impl<B: Backend> EncodableBlock<B> for Rope<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
let qkv_array = qkv_binding[0].borrow();
@@ -95,7 +91,8 @@ impl<B: Backend> EncodableBlock<B> for Rope<B> {
num_groups as u32,
suffix_length as u32,
rope_max_seq_len as u32,
- encoder,
- )
+ command_buffer,
+ );
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index 779c3fcd2..2b4901f0b 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::sampling::{ArgmaxStrategy, SamplingError, SamplingKernel},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -45,12 +45,12 @@ impl<B: Backend> Sampling<B> {
}
impl<B: Backend> EncodableBlock<B> for Sampling<B> {
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");
let logits_binding = state.arrays(&[ArrayId::Logits]);
@@ -59,7 +59,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {
let logits_shape = logits.shape();
let batch_size = state.sampling_length();
if batch_size == 0 {
- return;
+ return Ok(());
}
let sampling_start = state.sampling_start();
let vocab_size = logits_shape[1];
@@ -85,7 +85,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {
let seeds_buf_borrow = seeds_buf_rc.borrow();
let output_buf_rc = output_buffer_ref.buffer();
let mut output_buf_borrow = output_buf_rc.borrow_mut();
- if let Err(e) = self.kernel.encode_with_encoder(
+ if let Err(e) = self.kernel.encode(
logits_buf_borrow.deref_mut(),
Some(seeds_buf_borrow.deref()),
seeds_offset,
@@ -95,9 +95,10 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {
sampling_method,
batch_size,
vocab_size,
- encoder,
+ command_buffer,
) {
panic!("Sampling encoding failed: {:?}", e);
}
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index c6e272ccc..75534176f 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -103,48 +103,6 @@ impl<B: Backend> ShortConvMixer<B> {
}
impl<B: Backend> ShortConvMixer<B> {
- fn encode_pipeline(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
- ) -> Result<(), B::Error> {
- let active_suffix_length = state.active_suffix_length();
- if active_suffix_length == 0 {
- return Ok(());
- }
-
- self.in_projection.encode(state, parameters, command_buffer)?;
-
- command_buffer.with_compute_encoder(|encoder| self.run_conv(state, encoder, active_suffix_length));
-
- self.out_projection.encode(state, parameters, command_buffer)?;
-
- if parameters.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
- }
- Ok(())
- }
-
- fn encode_pipeline_with_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- encoder: &mut B::ComputeEncoder,
- parameters: &EncodingParameters<B>,
- ) {
- let active_suffix_length = state.active_suffix_length();
- if active_suffix_length == 0 {
- return;
- }
-
- self.in_projection.encode_with_shared_encoder(state, parameters, encoder);
-
- self.run_conv(state, encoder, active_suffix_length);
-
- self.out_projection.encode_with_shared_encoder(state, parameters, encoder);
- }
-
fn clear_suffix_state_valid_range(
&self,
state: &ForwardPassState<B>,
@@ -174,7 +132,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut B::ComputeEncoder,
+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
active_suffix_length: usize,
) {
self.clear_suffix_state_valid_range(state);
@@ -213,7 +171,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_prefill_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut B::ComputeEncoder,
+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -270,7 +228,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_trie_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut B::ComputeEncoder,
+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
sampling_start: usize,
trie_len: usize,
) {
@@ -329,7 +287,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_decode_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut B::ComputeEncoder,
+ compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -353,7 +311,7 @@ impl<B: Backend> ShortConvMixer<B> {
in_proj.buffer().borrow().deref(),
decode_weight_buf_borrow.deref(),
decode_bias_buf_borrow.as_deref(),
- None::<&B::NativeBuffer>,
+ None::<&B::Buffer>,
out.buffer().borrow_mut().deref_mut(),
conv_state.buffer().borrow_mut().deref_mut(),
suffix_length as u32,
@@ -370,33 +328,20 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- command_buffer: &mut B::CommandBuffer,
+ parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- if self.supports_shared_encoder() {
- command_buffer
- .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));
-
- if parameters.wait_until_completed {
- command_buffer.submit();
- command_buffer.wait_until_completed()?;
- }
- } else {
- self.encode_pipeline(state, parameters, command_buffer)?;
+ let active_suffix_length = state.active_suffix_length();
+ if active_suffix_length == 0 {
+ return Ok(());
}
- Ok(())
- }
- fn supports_shared_encoder(&self) -> bool {
- self.in_projection.supports_shared_encoder() && self.out_projection.supports_shared_encoder()
- }
+ self.in_projection.encode(state, parameters, command_buffer)?;
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
- self.encode_pipeline_with_encoder(state, encoder, parameters);
+ self.run_conv(state, command_buffer, active_suffix_length);
+
+ self.out_projection.encode(state, parameters, command_buffer)?;
+
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rs
index 961df3955..627208b12 100644
--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs
@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{Kernels, TensorAddSwapKernel},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -32,16 +32,12 @@ impl<B: Backend> TensorAddSwap<B> {
}
impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
@@ -54,7 +50,8 @@ impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {
skip_array.buffer().borrow_mut().deref_mut(),
main_array.buffer().borrow_mut().deref_mut(),
length as u32,
- encoder,
+ command_buffer,
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rs
index c8ab574ff..d82361ead 100644
--- a/crates/uzu/src/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/encodable_block/tensor_copy.rs
@@ -6,7 +6,7 @@ use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::{Kernels, TensorCopyKernel},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -32,16 +32,12 @@ impl<B: Backend> TensorCopy<B> {
}
impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
+ fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters<B>,
- encoder: &mut B::ComputeEncoder,
- ) {
+ _parameters: &EncodingParameters,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error> {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
@@ -54,7 +50,8 @@ impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {
source_array.buffer().borrow().deref(),
destination_array.buffer().borrow_mut().deref_mut(),
length as u32,
- encoder,
+ command_buffer,
);
+ Ok(())
}
}
diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
index e3485cd81..8a88a6741 100644
--- a/crates/uzu/src/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -5,7 +5,7 @@ use std::{
use crate::{
array::{Array, ArrayContextExt},
- backends::common::{Backend, kernel::kv_cache_update::KVCacheUpdate},
+ backends::common::{Backend, CommandBuffer, kernel::kv_cache_update::KVCacheUpdate},
config::DecoderLayerType,
forward_pass::{
kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},
@@ -276,7 +276,7 @@ impl<B: Backend> CacheLayers<B> {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
kv_cache_update: &KVCacheUpdate<B>,
) {
let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);
diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rs
index def9925c7..254bc84d8 100644
--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs
@@ -1,7 +1,7 @@
use crate::{
array::{Array, ArrayCell, ArrayContextExt},
backends::common::{
- Backend,
+ Backend, CommandBuffer,
kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},
},
utils::attention::fill_attention_bias,
@@ -163,7 +163,7 @@ impl<B: Backend> KVCacheLayer<B> {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
kv_cache_update: &KVCacheUpdate<B>,
) {
match &mut self.state {
@@ -213,7 +213,7 @@ impl<B: Backend> KVCacheLayer<B> {
&self,
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
kv_cache_update: &KVCacheUpdate<B>,
) {
if source_indices == destination_indices {
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index b78a8c777..c04a18ad9 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -10,7 +10,7 @@ use crate::forward_pass::traces::ActivationTrace;
use crate::{
DataType, DecoderConfig,
array::{Array, ArrayCell, ArrayCellExt},
- backends::common::{Backend, CommandBuffer, Context, CopyEncoder},
+ backends::common::{Backend, CommandBuffer, CommandBufferEncoding, Context},
forward_pass::{
cache_layers::CacheLayers,
model_shape::ModelShape,
@@ -157,8 +157,8 @@ impl<B: Backend> ForwardPassState<B> {
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
should_fill_attention_bias: bool,
- async_positions: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,
- async_seeds: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,
+ async_positions: Option<(Rc<RefCell<B::Buffer>>, usize)>,
+ async_seeds: Option<(Rc<RefCell<B::Buffer>>, usize)>,
) -> Self {
let suffix_length = token_ids.len();
assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");
@@ -777,7 +777,7 @@ impl<B: Backend> ForwardPassState<B> {
pub fn encode_copy_array(
&self,
- command_buffer: &mut B::CommandBuffer,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
source_array_id: ArrayId,
destination_array: RefCell<Array<B>>,
) {
@@ -791,8 +791,6 @@ impl<B: Backend> ForwardPassState<B> {
let copy_size_bytes = dst_borrow.size();
debug_assert_eq!(dst_borrow.size(), src_borrow.size());
- command_buffer.with_copy_encoder(|encoder| {
- encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);
- });
+ command_buffer.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);
}
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 7d59e7404..8a3685efd 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -19,7 +19,8 @@ use super::{
};
use crate::{
backends::common::{
- Backend, CommandBuffer, Context, NativeBuffer,
+ Backend, Buffer, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending, Context,
kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},
},
encodable_block::{EncodableBlock, EncodingParameters},
@@ -487,8 +488,6 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
*sm = Some(sampling_method);
}
- self.context.reset_command_buffer();
-
// Wait on previous pass if this is a continuation
if is_continuation {
self.context.command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
@@ -496,13 +495,13 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context
.executables
- .encode(&mut state, &EncodingParameters::new(false, false, false), &mut self.context.command_buffer)
+ .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
// Encode sampling
self.context
.gpu_sampler
- .encode(&mut state, &EncodingParameters::new(false, false, false), &mut self.context.command_buffer)
+ .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
// Copy sampled token: sampling_output → token_ids (for next pass)
@@ -515,19 +514,17 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let token_ids_buf_rc = token_ids_binding.buffer();
let mut token_ids_buf_borrow = token_ids_buf_rc.borrow_mut();
- self.context.command_buffer.with_compute_encoder(|encoder| {
- self.context.token_copy_sampled.encode(
- sampling_output_buf_borrow.deref(),
- token_ids_buf_borrow.deref_mut(),
- encoder,
- );
- let results_offset = slot * std::mem::size_of::<u32>();
- self.context.token_copy_results.encode(
- sampling_output_buf_borrow.deref(),
- (results_buffer.borrow_mut().deref_mut(), results_offset),
- encoder,
- );
- });
+ self.context.token_copy_sampled.encode(
+ sampling_output_buf_borrow.deref(),
+ token_ids_buf_borrow.deref_mut(),
+ &mut self.context.command_buffer,
+ );
+ let results_offset = slot * std::mem::size_of::<u32>();
+ self.context.token_copy_results.encode(
+ sampling_output_buf_borrow.deref(),
+ (results_buffer.borrow_mut().deref_mut(), results_offset),
+ &mut self.context.command_buffer,
+ );
// Scatter + register for all transformer layers
self.context.cache_layers.borrow_mut().update_after_acceptance(
@@ -538,26 +535,24 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
);
self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
- self.context.command_buffer.with_compute_encoder(|encoder| {
- if let Some(mask_update) = &self.context.mask_update {
- let updates: Vec<AttentionBiasUpdate> =
- self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
- for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {
- if let Some(update) = updates.iter().find(|u| &u.key == window_size) {
- if update.unmask_col >= 0 || update.mask_col >= 0 {
- let mask_buf_rc = mask_buffer.borrow().buffer();
- let mut mask_buf_borrow = mask_buf_rc.borrow_mut();
- mask_update.encode(
- mask_buf_borrow.deref_mut(),
- update.unmask_col,
- update.mask_col,
- encoder,
- );
- }
+ if let Some(mask_update) = &self.context.mask_update {
+ let updates: Vec<AttentionBiasUpdate> =
+ self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
+ for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {
+ if let Some(update) = updates.iter().find(|u| &u.key == window_size) {
+ if update.unmask_col >= 0 || update.mask_col >= 0 {
+ let mask_buf_rc = mask_buffer.borrow().buffer();
+ let mut mask_buf_borrow = mask_buf_rc.borrow_mut();
+ mask_update.encode(
+ mask_buf_borrow.deref_mut(),
+ update.unmask_col,
+ update.mask_col,
+ &mut self.context.command_buffer,
+ );
}
}
}
- });
+ }
// Signal event for next pass
let next_counter = current_counter + 1;
@@ -567,17 +562,18 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// Add completion handler
let results_buffer_ptr = results_buffer.borrow().cpu_ptr().as_ptr() as *const u32;
- let handler = move |result: Result<(), B::Error>| {
+ let handler = move |result: Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>| {
result.expect("async decoding forward pass completed with error");
let token = { unsafe { *results_buffer_ptr.add(slot) as u64 } };
on_complete(token);
};
self.context.command_buffer.add_completion_handler(handler);
- self.context.command_buffer.submit();
+ drop(token_ids_binding);
+ let pending = self.context.replace_command_buffer().end_encoding().submit();
if should_capture {
- self.context.command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
}
@@ -772,12 +768,12 @@ impl<B: Backend> LanguageModelGenerator<B> {
if let Some(_) = self.encoded_tasks.remove(&encoded_task_key) {
// Nothing
} else {
- self.context.reset_command_buffer();
+ self.context.replace_command_buffer();
task.build_encoded_task(
&mut self.context,
&mut state,
- &EncodingParameters::new(warmup, true, false),
+ &EncodingParameters::new(),
encoded_task_key.clone(),
)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
@@ -787,41 +783,34 @@ impl<B: Backend> LanguageModelGenerator<B> {
if !task.is_prefilling {
self.context
.gpu_sampler
- .encode(
- &mut state,
- &EncodingParameters::new(warmup, true, false),
- &mut self.context.command_buffer,
- )
+ .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
}
- self.context.command_buffer.submit();
+ let command_buffer = std::mem::replace(
+ &mut self.context.command_buffer,
+ self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),
+ );
+ let pending = command_buffer.end_encoding().submit();
if allow_pre_encode {
- let in_flight_command_buffer = self.context.reset_command_buffer();
-
let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);
let next_encoded_task = task
.build_encoded_task(
&mut self.context,
&mut state,
- &EncodingParameters::new(warmup, false, false).with_projection(1),
+ &EncodingParameters::new().with_projection(1),
next_task_key.clone(),
)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);
-
- in_flight_command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
- } else {
- self.context
- .command_buffer
- .wait_until_completed()
- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
}
+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+
let run_time = run_start.elapsed().as_secs_f64();
if should_capture {
self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
@@ -857,7 +846,8 @@ impl<B: Backend> LanguageModelGenerator<B> {
suffix_start: Option<usize>,
wait_until_completed: bool,
) -> Result<(), Error> {
- let mut command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer =
+ self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
{
let mut cache_layers = self.context.cache_layers.borrow_mut();
@@ -869,10 +859,10 @@ impl<B: Backend> LanguageModelGenerator<B> {
);
}
- command_buffer.submit();
+ let pending = command_buffer.end_encoding().submit();
if wait_until_completed {
- command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
}
Ok(())
}
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index b5d576b0d..f473fea09 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -9,7 +9,7 @@ use std::{
use crate::{
DataType,
backends::common::{
- Backend, Context, Kernels, NativeBuffer,
+ Backend, Buffer, CommandBuffer, CommandBufferInitial, Context, Kernels,
kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},
},
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
@@ -31,13 +31,13 @@ use crate::{
pub struct AsyncBuffers<B: Backend> {
/// Positions buffer: [max_tokens] i32
/// Pre-populated with [prefill_count, prefill_count+1, ...]
- pub positions: Rc<RefCell<B::NativeBuffer>>,
+ pub positions: Rc<RefCell<B::Buffer>>,
/// Seeds buffer: [max_tokens] u64
/// Pre-populated with deterministic seed sequence
- pub seeds: Rc<RefCell<B::NativeBuffer>>,
+ pub seeds: Rc<RefCell<B::Buffer>>,
/// Results buffer: [batch_size] u32
/// Each pass writes its sampled token to results[pass_idx % batch_size]
- pub results: Rc<RefCell<B::NativeBuffer>>,
+ pub results: Rc<RefCell<B::Buffer>>,
/// Event for GPU-side synchronization between passes
pub event: B::Event,
/// Current event counter (pass N waits on N, signals N+1)
@@ -122,7 +122,7 @@ impl<B: Backend> AsyncBuffers<B> {
pub struct LanguageModelGeneratorContext<B: Backend> {
pub context: Rc<B::Context>,
- pub command_buffer: B::CommandBuffer,
+ pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,
pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
@@ -151,7 +151,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
decoding_config: &DecodingConfig,
) -> Result<Self, Error> {
let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -247,10 +247,10 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
Ok(context)
}
- pub fn reset_command_buffer(&mut self) -> B::CommandBuffer {
+ pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {
std::mem::replace(
&mut self.command_buffer,
- self.context.create_command_buffer().expect("Failed to create command buffer"),
+ self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),
)
}
}
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 5063c8570..3df98449f 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -2,18 +2,18 @@ use std::mem::size_of;
use super::language_model_generator_context::LanguageModelGeneratorContext;
use crate::{
- backends::common::{Backend, Context, NativeBuffer},
+ backends::common::{Backend, Buffer, Context},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::ForwardPassState,
};
pub struct LanguageModelGeneratorEncodedTask<B: Backend> {
pub key: String,
- predicate_buffer: B::NativeBuffer,
+ predicate_buffer: B::Buffer,
}
impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {
- pub fn predicate_buffer(&self) -> &B::NativeBuffer {
+ pub fn predicate_buffer(&self) -> &B::Buffer {
&self.predicate_buffer
}
@@ -111,7 +111,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
&self,
context: &mut LanguageModelGeneratorContext<B>,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters<B>,
+ parameters: &EncodingParameters,
key: String,
) -> Result<LanguageModelGeneratorEncodedTask<B>, B::Error> {
context.executables.encode(state, parameters, &mut context.command_buffer)?;
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 164c62994..027b25f73 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -18,7 +18,10 @@ use num_traits::NumCast;
use crate::{
ArrayElement, DataType,
array::Array,
- backends::common::{Backend, CommandBuffer, Context, kernel::kv_cache_update::KVCacheUpdate},
+ backends::common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ kernel::kv_cache_update::KVCacheUpdate,
+ },
classifier::Classifier,
config::ModelMetadata,
encodable_block::{EncodableBlock, EncodingParameters, Sampling},
@@ -270,13 +273,14 @@ impl<B: Backend> TraceValidator<B> {
None,
);
- let mut command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer =
+ ctx.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
ctx.executables
- .encode(&mut state, &EncodingParameters::new(false, false, false), &mut command_buffer)
+ .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- command_buffer.submit();
- command_buffer.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ let pending = command_buffer.end_encoding().submit();
+ pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
let traces = state.traces().clone();
let data_type = ctx.model_shape.activation_data_type();
diff --git a/crates/uzu/tests/allocator_test/allocation_stats.rs b/crates/uzu/tests/allocator_test/allocation_stats.rs
deleted file mode 100644
index cf151c835..000000000
--- a/crates/uzu/tests/allocator_test/allocation_stats.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-use std::time::Duration;
-
-#[derive(Clone, Copy, Default)]
-pub struct AllocationStats {
- pub total_allocs: usize,
- pub total_frees: usize,
- pub total_alloc_time_ns: u128,
- pub total_free_time_ns: u128,
- pub total_bytes_requested: usize,
-}
-
-impl AllocationStats {
- pub fn record_alloc(
- &mut self,
- size: usize,
- duration: Duration,
- ) {
- self.total_allocs += 1;
- self.total_alloc_time_ns += duration.as_nanos();
- self.total_bytes_requested += size;
- }
-
- pub fn record_free(
- &mut self,
- duration: Duration,
- ) {
- self.total_frees += 1;
- self.total_free_time_ns += duration.as_nanos();
- }
-
- pub fn avg_alloc_time_us(&self) -> f64 {
- if self.total_allocs == 0 {
- 0.0
- } else {
- self.total_alloc_time_ns as f64 / self.total_allocs as f64 / 1000.0
- }
- }
-
- pub fn avg_free_time_us(&self) -> f64 {
- if self.total_frees == 0 {
- 0.0
- } else {
- self.total_free_time_ns as f64 / self.total_frees as f64 / 1000.0
- }
- }
-}
diff --git a/crates/uzu/tests/allocator_test/allocator_trait.rs b/crates/uzu/tests/allocator_test/allocator_trait.rs
deleted file mode 100644
index db5eb325d..000000000
--- a/crates/uzu/tests/allocator_test/allocator_trait.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-pub trait AllocatorTrait {
- type Buffer;
-
- fn alloc(
- &self,
- size: usize,
- ) -> Self::Buffer;
- fn free(
- &self,
- buffer: Self::Buffer,
- );
- fn peak_memory(&self) -> usize;
- fn cache_memory(&self) -> usize;
- fn active_memory(&self) -> usize;
-}
diff --git a/crates/uzu/tests/allocator_test/caching_allocator.rs b/crates/uzu/tests/allocator_test/caching_allocator.rs
deleted file mode 100644
index 31773c871..000000000
--- a/crates/uzu/tests/allocator_test/caching_allocator.rs
+++ /dev/null
@@ -1,51 +0,0 @@
-use std::rc::Rc;
-
-use uzu::backends::{
- common::{Backend, Buffer, BufferLifetime, Context},
- metal::Metal,
-};
-
-use super::allocator_trait::AllocatorTrait;
-
-pub struct CachingAllocator {
- pub context: Rc<<Metal as Backend>::Context>,
-}
-
-impl CachingAllocator {
- pub fn new() -> Self {
- let context = <Metal as Backend>::Context::new().expect("Failed to create MetalContext");
- Self {
- context,
- }
- }
-}
-
-impl AllocatorTrait for CachingAllocator {
- type Buffer = Buffer<Metal>;
-
- fn alloc(
- &self,
- size: usize,
- ) -> Self::Buffer {
- self.context.allocator().alloc(BufferLifetime::Scratch, size).expect("Allocation failed")
- }
-
- fn free(
- &self,
- buffer: Self::Buffer,
- ) {
- drop(buffer);
- }
-
- fn peak_memory(&self) -> usize {
- self.context.allocator().peak_memory()
- }
-
- fn cache_memory(&self) -> usize {
- self.context.allocator().cache_memory()
- }
-
- fn active_memory(&self) -> usize {
- self.context.allocator().active_memory()
- }
-}
diff --git a/crates/uzu/tests/allocator_test/device_allocator.rs b/crates/uzu/tests/allocator_test/device_allocator.rs
deleted file mode 100644
index 0ee9a33ac..000000000
--- a/crates/uzu/tests/allocator_test/device_allocator.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-use std::cell::Cell;
-
-use metal::{MTLBuffer, MTLDevice, MTLDeviceExt, MTLResourceOptions};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::allocator_trait::AllocatorTrait;
-
-pub struct DeviceAllocator {
- device: Retained<ProtocolObject<dyn MTLDevice>>,
- active_memory: Cell<usize>,
- peak_memory: Cell<usize>,
-}
-
-impl DeviceAllocator {
- pub fn new() -> Self {
- let device = <dyn MTLDevice>::system_default().expect("No Metal device");
-
- Self {
- device,
- active_memory: Cell::new(0),
- peak_memory: Cell::new(0),
- }
- }
-}
-
-impl AllocatorTrait for DeviceAllocator {
- type Buffer = Retained<ProtocolObject<dyn MTLBuffer>>;
-
- fn alloc(
- &self,
- size: usize,
- ) -> Self::Buffer {
- let buffer =
- self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");
-
- let buf_size = buffer.length();
- let new_active = self.active_memory.get() + buf_size;
- self.active_memory.set(new_active);
-
- if new_active > self.peak_memory.get() {
- self.peak_memory.set(new_active);
- }
-
- buffer
- }
-
- fn free(
- &self,
- buffer: Self::Buffer,
- ) {
- let size = buffer.length();
- self.active_memory.set(self.active_memory.get().saturating_sub(size));
- drop(buffer);
- }
-
- fn peak_memory(&self) -> usize {
- self.peak_memory.get()
- }
-
- fn cache_memory(&self) -> usize {
- 0
- }
-
- fn active_memory(&self) -> usize {
- self.active_memory.get()
- }
-}
diff --git a/crates/uzu/tests/allocator_test/heap_allocator.rs b/crates/uzu/tests/allocator_test/heap_allocator.rs
deleted file mode 100644
index 9376baaae..000000000
--- a/crates/uzu/tests/allocator_test/heap_allocator.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-use std::cell::Cell;
-
-use metal::{MTLBuffer, MTLDevice, MTLDeviceExt, MTLHeap, MTLHeapDescriptor, MTLResourceOptions, MTLStorageMode};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::allocator_trait::AllocatorTrait;
-
-pub struct MTLHeapAllocator {
- heap: Retained<ProtocolObject<dyn MTLHeap>>,
- peak_used: Cell<usize>,
- active_memory: Cell<usize>,
-}
-
-impl MTLHeapAllocator {
- pub fn new(heap_size: usize) -> Self {
- let device = <dyn MTLDevice>::system_default().expect("No Metal device");
-
- let descriptor = MTLHeapDescriptor::new();
- descriptor.set_size(heap_size);
- descriptor.set_storage_mode(MTLStorageMode::Shared);
-
- let heap = device.new_heap_with_descriptor(&descriptor).expect("Failed to create heap");
-
- Self {
- heap,
- peak_used: Cell::new(0),
- active_memory: Cell::new(0),
- }
- }
-}
-
-impl AllocatorTrait for MTLHeapAllocator {
- type Buffer = Retained<ProtocolObject<dyn MTLBuffer>>;
-
- fn alloc(
- &self,
- size: usize,
- ) -> Self::Buffer {
- let buffer = self
- .heap
- .new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer from heap");
-
- let buf_size = buffer.length();
- let new_active = self.active_memory.get() + buf_size;
- self.active_memory.set(new_active);
-
- let used = self.heap.used_size();
- if used > self.peak_used.get() {
- self.peak_used.set(used);
- }
-
- buffer
- }
-
- fn free(
- &self,
- buffer: Self::Buffer,
- ) {
- let size = buffer.length();
- self.active_memory.set(self.active_memory.get().saturating_sub(size));
- drop(buffer);
- }
-
- fn peak_memory(&self) -> usize {
- self.peak_used.get()
- }
-
- fn cache_memory(&self) -> usize {
- 0
- }
-
- fn active_memory(&self) -> usize {
- self.active_memory.get()
- }
-}
diff --git a/crates/uzu/tests/allocator_test/inference_simulation.rs b/crates/uzu/tests/allocator_test/inference_simulation.rs
deleted file mode 100644
index 812108b1e..000000000
--- a/crates/uzu/tests/allocator_test/inference_simulation.rs
+++ /dev/null
@@ -1,197 +0,0 @@
-use std::time::Instant;
-
-use super::{allocation_stats::AllocationStats, allocator_trait::AllocatorTrait, model_config::ModelConfig};
-
-const F16_SIZE: usize = 2;
-
-#[derive(Clone, Copy, Debug)]
-pub struct MemorySnapshot {
- pub step: usize,
- pub active_memory: usize,
- pub cache_memory: usize,
- pub peak_memory: usize,
-}
-
-pub struct InferenceSimulation<'a, A: AllocatorTrait> {
- allocator: &'a A,
- config: ModelConfig,
- pub stats: AllocationStats,
- pub current_prefix_len: usize,
- step_counter: usize,
-}
-
-impl<'a, A: AllocatorTrait> InferenceSimulation<'a, A> {
- pub fn new(
- allocator: &'a A,
- config: ModelConfig,
- ) -> Self {
- Self {
- allocator,
- config,
- stats: AllocationStats::default(),
- current_prefix_len: 0,
- step_counter: 0,
- }
- }
-
- pub fn capture_snapshot(&self) -> MemorySnapshot {
- MemorySnapshot {
- step: self.step_counter,
- active_memory: self.allocator.active_memory(),
- cache_memory: self.allocator.cache_memory(),
- peak_memory: self.allocator.peak_memory(),
- }
- }
-
- fn alloc_scratch(
- &mut self,
- size: usize,
- ) -> A::Buffer {
- let start = Instant::now();
- let result = self.allocator.alloc(size);
- self.stats.record_alloc(size, start.elapsed());
- result
- }
-
- fn free(
- &mut self,
- buffer: A::Buffer,
- ) {
- let start = Instant::now();
- self.allocator.free(buffer);
- self.stats.record_free(start.elapsed());
- }
-
- fn simulate_embedding_kernel(
- &mut self,
- batch_size: usize,
- ) -> A::Buffer {
- let size = batch_size * self.config.model_dim * F16_SIZE;
- self.alloc_scratch(size)
- }
-
- fn simulate_attention_kernel(
- &mut self,
- batch_size: usize,
- ) -> Vec<A::Buffer> {
- let mut buffers = Vec::new();
-
- let qkv_size = batch_size * self.config.qkv_dim() * F16_SIZE;
- buffers.push(self.alloc_scratch(qkv_size));
-
- let bias_size = batch_size * (batch_size + self.current_prefix_len) * F16_SIZE;
- buffers.push(self.alloc_scratch(bias_size));
-
- let rotated_q_size = self.config.num_heads * batch_size * self.config.head_dim * F16_SIZE;
- buffers.push(self.alloc_scratch(rotated_q_size));
-
- let rotated_k_size = self.config.num_groups * batch_size * self.config.head_dim * F16_SIZE;
- buffers.push(self.alloc_scratch(rotated_k_size));
-
- let attn_out_size = batch_size * self.config.num_heads * self.config.head_dim * F16_SIZE;
- buffers.push(self.alloc_scratch(attn_out_size));
-
- buffers
- }
-
- fn simulate_mlp_kernel(
- &mut self,
- batch_size: usize,
- ) -> Vec<A::Buffer> {
- let mut buffers = Vec::new();
-
- let up_size = batch_size * 2 * self.config.hidden_dim * F16_SIZE;
- buffers.push(self.alloc_scratch(up_size));
-
- let hidden_size = batch_size * self.config.hidden_dim * F16_SIZE;
- buffers.push(self.alloc_scratch(hidden_size));
-
- buffers
- }
-
- fn simulate_logits_kernel(
- &mut self,
- batch_size: usize,
- ) -> A::Buffer {
- let size = batch_size * self.config.vocab_size * F16_SIZE;
- self.alloc_scratch(size)
- }
-
- fn simulate_forward_pass(
- &mut self,
- batch_size: usize,
- ) {
- self.step_counter += 1;
-
- let main = self.simulate_embedding_kernel(batch_size);
-
- for _layer in 0..self.config.num_layers {
- let attn_buffers = self.simulate_attention_kernel(batch_size);
- for buf in attn_buffers {
- self.free(buf);
- }
-
- let mlp_buffers = self.simulate_mlp_kernel(batch_size);
- for buf in mlp_buffers {
- self.free(buf);
- }
- }
-
- let logits = self.simulate_logits_kernel(batch_size);
- self.free(logits);
- self.free(main);
- }
-
- pub fn run_prefill(
- &mut self,
- prompt_length: usize,
- step_size: usize,
- ) -> Vec<MemorySnapshot> {
- let mut snapshots = Vec::new();
- let mut remaining = prompt_length;
-
- while remaining > 0 {
- let batch_size = remaining.min(step_size);
- self.simulate_forward_pass(batch_size);
- self.current_prefix_len += batch_size;
- remaining = remaining.saturating_sub(step_size);
- snapshots.push(self.capture_snapshot());
- }
-
- snapshots
- }
-
- pub fn run_generation(
- &mut self,
- num_tokens: usize,
- ) -> Vec<MemorySnapshot> {
- let mut snapshots = Vec::new();
-
- for _ in 0..num_tokens {
- self.simulate_forward_pass(1);
- self.current_prefix_len += 1;
- snapshots.push(self.capture_snapshot());
- }
-
- snapshots
- }
-
- pub fn run_generation_with_sampling(
- &mut self,
- num_tokens: usize,
- snapshot_interval: usize,
- ) -> Vec<MemorySnapshot> {
- let mut snapshots = Vec::new();
-
- for i in 0..num_tokens {
- self.simulate_forward_pass(1);
- self.current_prefix_len += 1;
-
- if i % snapshot_interval == 0 || i == num_tokens - 1 {
- snapshots.push(self.capture_snapshot());
- }
- }
-
- snapshots
- }
-}
diff --git a/crates/uzu/tests/allocator_test/main.rs b/crates/uzu/tests/allocator_test/main.rs
deleted file mode 100644
index 0d0741726..000000000
--- a/crates/uzu/tests/allocator_test/main.rs
+++ /dev/null
@@ -1,270 +0,0 @@
-#![cfg(target_os = "macos")]
-
-mod allocation_stats;
-mod allocator_trait;
-mod caching_allocator;
-mod device_allocator;
-mod heap_allocator;
-mod inference_simulation;
-mod model_config;
-mod simulation_result;
-
-use caching_allocator::CachingAllocator;
-use device_allocator::DeviceAllocator;
-use heap_allocator::MTLHeapAllocator;
-use model_config::ModelConfig;
-use simulation_result::{run_long_simulation, run_simulation, run_simulation_with_config};
-use uzu::backends::common::Context;
-
-const KB: f64 = 1024.0;
-const MB: f64 = 1024.0 * 1024.0;
-
-fn print_comparison(
- name: &str,
- uzu_result: &simulation_result::SimulationResult,
- heap_result: &simulation_result::SimulationResult,
- device_result: &simulation_result::SimulationResult,
-) {
- println!("\n=== {} ===", name);
- println!("{:<20} {:>14} {:>14} {:>14}", "Metric", "UZU Caching", "Heap", "Device");
- println!("{:-<62}", "");
-
- println!(
- "{:<20} {:>14} {:>14} {:>14}",
- "Allocations", uzu_result.stats.total_allocs, heap_result.stats.total_allocs, device_result.stats.total_allocs
- );
-
- println!(
- "{:<20} {:>14} {:>14} {:>14}",
- "Frees", uzu_result.stats.total_frees, heap_result.stats.total_frees, device_result.stats.total_frees
- );
-
- println!(
- "{:<20} {:>13.2}µs {:>13.2}µs {:>13.2}µs",
- "Avg alloc time",
- uzu_result.stats.avg_alloc_time_us(),
- heap_result.stats.avg_alloc_time_us(),
- device_result.stats.avg_alloc_time_us()
- );
-
- println!(
- "{:<20} {:>13.2}µs {:>13.2}µs {:>13.2}µs",
- "Avg free time",
- uzu_result.stats.avg_free_time_us(),
- heap_result.stats.avg_free_time_us(),
- device_result.stats.avg_free_time_us()
- );
-
- println!(
- "{:<20} {:>12.2} MB {:>12.2} MB {:>12.2} MB",
- "Bytes requested",
- uzu_result.stats.total_bytes_requested as f64 / MB,
- heap_result.stats.total_bytes_requested as f64 / MB,
- device_result.stats.total_bytes_requested as f64 / MB
- );
-
- println!(
- "{:<20} {:>12.2} MB {:>12.2} MB {:>12.2} MB",
- "Peak memory",
- uzu_result.peak_memory_bytes as f64 / MB,
- heap_result.peak_memory_bytes as f64 / MB,
- device_result.peak_memory_bytes as f64 / MB
- );
-
- println!(
- "{:<20} {:>12.2} KB {:>12.2} KB {:>12.2} KB",
- "Final cache",
- uzu_result.cache_memory_bytes as f64 / KB,
- heap_result.cache_memory_bytes as f64 / KB,
- device_result.cache_memory_bytes as f64 / KB
- );
-
- println!();
-
- let device_time = device_result.stats.avg_alloc_time_us().max(0.001);
- let uzu_speedup = device_time / uzu_result.stats.avg_alloc_time_us().max(0.001);
- let heap_speedup = device_time / heap_result.stats.avg_alloc_time_us().max(0.001);
-
- println!("Alloc speedup vs device: UZU {:.1}x, Heap {:.1}x", uzu_speedup, heap_speedup);
-}
-
-fn print_memory_over_time(
- name: &str,
- snapshots: &[inference_simulation::MemorySnapshot],
- sample_count: usize,
-) {
- if snapshots.is_empty() {
- return;
- }
-
- println!("\n--- {} Memory Over Time (sampled) ---", name);
- println!("{:>8} {:>12} {:>12} {:>12}", "Step", "Active", "Cache", "Peak");
-
- let step = (snapshots.len() / sample_count).max(1);
- for (i, snapshot) in snapshots.iter().enumerate() {
- if i % step == 0 || i == snapshots.len() - 1 {
- println!(
- "{:>8} {:>10.2}MB {:>10.2}MB {:>10.2}MB",
- snapshot.step,
- snapshot.active_memory as f64 / MB,
- snapshot.cache_memory as f64 / MB,
- snapshot.peak_memory as f64 / MB
- );
- }
- }
-}
-
-#[test]
-fn test_allocator_comparison() {
- let prompt_length = 256;
- let generate_length = 2048;
- let prefill_step_size = 512;
-
- println!("\n========================================");
- println!(" Short Conversation Test");
- println!(" {}p + {}g tokens", prompt_length, generate_length);
- println!("========================================");
-
- let uzu_allocator = CachingAllocator::new();
- let uzu_result = run_simulation(&uzu_allocator, prompt_length, generate_length, prefill_step_size);
-
- let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);
- let heap_result = run_simulation(&heap_allocator, prompt_length, generate_length, prefill_step_size);
-
- let device_allocator = DeviceAllocator::new();
- let device_result = run_simulation(&device_allocator, prompt_length, generate_length, prefill_step_size);
-
- print_comparison("Short Conversation Results", &uzu_result, &heap_result, &device_result);
-
- assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");
- assert_eq!(
- heap_result.stats.total_allocs, heap_result.stats.total_frees,
- "Heap allocator: allocs should equal frees"
- );
- assert_eq!(
- device_result.stats.total_allocs, device_result.stats.total_frees,
- "Device allocator: allocs should equal frees"
- );
-}
-
-#[test]
-fn test_long_conversation_comparison() {
- let num_conversations = 10;
- let avg_prompt_length = 512;
- let avg_generate_length = 1024;
- let prefill_step_size = 512;
- let snapshot_interval = 100;
-
- println!("\n========================================");
- println!(" Long Conversation Test");
- println!(" {} conversations, ~{}p + ~{}g tokens each", num_conversations, avg_prompt_length, avg_generate_length);
- println!("========================================");
-
- let gc_limit = 256 * 1024 * 1024;
- let max_pool = 128 * 1024 * 1024;
-
- let uzu_allocator = CachingAllocator::new();
- uzu_allocator.context.allocator().set_gc_limit(gc_limit);
- uzu_allocator.context.allocator().set_max_pool_size(max_pool);
-
- let uzu_result = run_long_simulation(
- &uzu_allocator,
- num_conversations,
- avg_prompt_length,
- avg_generate_length,
- prefill_step_size,
- snapshot_interval,
- );
-
- let heap_allocator = MTLHeapAllocator::new(1024 * 1024 * 1024);
- let heap_result = run_long_simulation(
- &heap_allocator,
- num_conversations,
- avg_prompt_length,
- avg_generate_length,
- prefill_step_size,
- snapshot_interval,
- );
-
- let device_allocator = DeviceAllocator::new();
- let device_result = run_long_simulation(
- &device_allocator,
- num_conversations,
- avg_prompt_length,
- avg_generate_length,
- prefill_step_size,
- snapshot_interval,
- );
-
- print_comparison("Long Conversation Results", &uzu_result, &heap_result, &device_result);
-
- print_memory_over_time("UZU Caching", &uzu_result.snapshots, 15);
- print_memory_over_time("Heap", &heap_result.snapshots, 15);
- print_memory_over_time("Device (baseline)", &device_result.snapshots, 15);
-
- println!("\n--- Memory Efficiency Summary ---");
- println!(
- "UZU peak/requested ratio: {:.2}%",
- (uzu_result.peak_memory_bytes as f64 / uzu_result.stats.total_bytes_requested as f64) * 100.0
- );
- println!(
- "Heap peak/requested ratio: {:.2}%",
- (heap_result.peak_memory_bytes as f64 / heap_result.stats.total_bytes_requested as f64) * 100.0
- );
- println!(
- "Device peak/requested ratio: {:.2}%",
- (device_result.peak_memory_bytes as f64 / device_result.stats.total_bytes_requested as f64) * 100.0
- );
-
- assert_eq!(uzu_result.stats.total_allocs, uzu_result.stats.total_frees, "UZU allocator: allocs should equal frees");
- assert_eq!(
- device_result.stats.total_allocs, device_result.stats.total_frees,
- "Device allocator: allocs should equal frees"
- );
-}
-
-#[test]
-fn test_multi_model_comparison() {
- let prompt_length = 256;
- let generate_length = 512;
- let prefill_step_size = 256;
-
- println!("\n========================================");
- println!(" Multi-Model Comparison Test");
- println!(" {}p + {}g tokens per model", prompt_length, generate_length);
- println!("========================================");
-
- println!("\n{:<15} {:>12} {:>12} {:>12}", "Model", "UZU (µs)", "Heap (µs)", "Device (µs)");
- println!("{:-<55}", "");
-
- for config in ModelConfig::all_models() {
- let uzu_allocator = CachingAllocator::new();
- let uzu_result =
- run_simulation_with_config(&uzu_allocator, config, prompt_length, generate_length, prefill_step_size);
-
- let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);
- let heap_result =
- run_simulation_with_config(&heap_allocator, config, prompt_length, generate_length, prefill_step_size);
-
- let device_allocator = DeviceAllocator::new();
- let device_result =
- run_simulation_with_config(&device_allocator, config, prompt_length, generate_length, prefill_step_size);
-
- println!(
- "{:<15} {:>12.2} {:>12.2} {:>12.2}",
- config.name,
- uzu_result.stats.avg_alloc_time_us(),
- heap_result.stats.avg_alloc_time_us(),
- device_result.stats.avg_alloc_time_us()
- );
-
- assert_eq!(
- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,
- "{}: UZU allocs should equal frees",
- config.name
- );
- }
-
- println!();
- println!("Summary: Lower allocation time is better.");
-}
diff --git a/crates/uzu/tests/allocator_test/mod.rs b/crates/uzu/tests/allocator_test/mod.rs
deleted file mode 100644
index 32883483e..000000000
--- a/crates/uzu/tests/allocator_test/mod.rs
+++ /dev/null
@@ -1,279 +0,0 @@
-#![cfg(target_os = "macos")]
-
-mod allocation_stats;
-mod allocator_trait;
-mod caching_allocator;
-mod device_allocator;
-mod heap_allocator;
-mod inference_simulation;
-mod mlx_allocator;
-mod model_config;
-mod simulation_result;
-
-use caching_allocator::CachingAllocator;
-use device_allocator::DeviceAllocator;
-use heap_allocator::MTLHeapAllocator;
-use mlx_allocator::MlxAllocator;
-use simulation_result::{run_long_simulation, run_simulation};
-use uzu::backends::common::Context;
-
-const KB: f64 = 1024.0;
-const MB: f64 = 1024.0 * 1024.0;
-const GB: f64 = 1024.0 * 1024.0 * 1024.0;
-
-fn print_three_way_comparison(
- name: &str,
- uzu_result: &simulation_result::SimulationResult,
- mlx_result: &simulation_result::SimulationResult,
- heap_result: &simulation_result::SimulationResult,
-) {
- println!("\n=== {} ===", name);
- println!(
- "{:<25} {:>15} {:>15} {:>15}",
- "Metric", "UZU Caching", "MLX-style LRU", "Heap (baseline)"
- );
- println!("{:-<72}", "");
-
- println!(
- "{:<25} {:>15} {:>15} {:>15}",
- "Allocations",
- uzu_result.stats.total_allocs,
- mlx_result.stats.total_allocs,
- heap_result.stats.total_allocs
- );
-
- println!(
- "{:<25} {:>15} {:>15} {:>15}",
- "Frees",
- uzu_result.stats.total_frees,
- mlx_result.stats.total_frees,
- heap_result.stats.total_frees
- );
-
- println!(
- "{:<25} {:>14.2}µs {:>14.2}µs {:>14.2}µs",
- "Avg alloc time",
- uzu_result.stats.avg_alloc_time_us(),
- mlx_result.stats.avg_alloc_time_us(),
- heap_result.stats.avg_alloc_time_us()
- );
-
- println!(
- "{:<25} {:>14.2}µs {:>14.2}µs {:>14.2}µs",
- "Avg free time",
- uzu_result.stats.avg_free_time_us(),
- mlx_result.stats.avg_free_time_us(),
- heap_result.stats.avg_free_time_us()
- );
-
- println!(
- "{:<25} {:>13.2} MB {:>13.2} MB {:>13.2} MB",
- "Bytes requested",
- uzu_result.stats.total_bytes_requested as f64 / MB,
- mlx_result.stats.total_bytes_requested as f64 / MB,
- heap_result.stats.total_bytes_requested as f64 / MB
- );
-
- println!(
- "{:<25} {:>13.2} MB {:>13.2} MB {:>13.2} MB",
- "Peak memory",
- uzu_result.peak_memory_bytes as f64 / MB,
- mlx_result.peak_memory_bytes as f64 / MB,
- heap_result.peak_memory_bytes as f64 / MB
- );
-
- println!(
- "{:<25} {:>13.2} KB {:>13.2} KB {:>13.2} KB",
- "Final cache",
- uzu_result.cache_memory_bytes as f64 / KB,
- mlx_result.cache_memory_bytes as f64 / KB,
- heap_result.cache_memory_bytes as f64 / KB
- );
-
- println!();
-
- let uzu_speedup = heap_result.stats.avg_alloc_time_us()
- / uzu_result.stats.avg_alloc_time_us().max(0.001);
- let mlx_speedup = heap_result.stats.avg_alloc_time_us()
- / mlx_result.stats.avg_alloc_time_us().max(0.001);
-
- println!(
- "Alloc speedup vs heap: UZU {:.1}x, MLX {:.1}x",
- uzu_speedup, mlx_speedup
- );
-}
-
-fn print_memory_over_time(
- name: &str,
- snapshots: &[inference_simulation::MemorySnapshot],
- sample_count: usize,
-) {
- if snapshots.is_empty() {
- return;
- }
-
- println!("\n--- {} Memory Over Time (sampled) ---", name);
- println!("{:>8} {:>12} {:>12} {:>12}", "Step", "Active", "Cache", "Peak");
-
- let step = (snapshots.len() / sample_count).max(1);
- for (i, snapshot) in snapshots.iter().enumerate() {
- if i % step == 0 || i == snapshots.len() - 1 {
- println!(
- "{:>8} {:>10.2}MB {:>10.2}MB {:>10.2}MB",
- snapshot.step,
- snapshot.active_memory as f64 / MB,
- snapshot.cache_memory as f64 / MB,
- snapshot.peak_memory as f64 / MB
- );
- }
- }
-}
-
-#[test]
-fn test_allocator_comparison() {
- let prompt_length = 256;
- let generate_length = 2048;
- let prefill_step_size = 512;
-
- println!("\n========================================");
- println!(" Short Conversation Test");
- println!(" {}p + {}g tokens", prompt_length, generate_length);
- println!("========================================");
-
- let uzu_allocator = CachingAllocator::new();
- let uzu_result = run_simulation(
- &uzu_allocator,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
-
- let mlx_allocator = MlxAllocator::new(512 * 1024 * 1024, 256 * 1024 * 1024);
- let mlx_result = run_simulation(
- &mlx_allocator,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
-
- let heap_allocator = MTLHeapAllocator::new(512 * 1024 * 1024);
- let heap_result = run_simulation(
- &heap_allocator,
- prompt_length,
- generate_length,
- prefill_step_size,
- );
-
- print_three_way_comparison(
- "Short Conversation Results",
- &uzu_result,
- &mlx_result,
- &heap_result,
- );
-
- assert_eq!(
- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,
- "UZU allocator: allocs should equal frees"
- );
- assert_eq!(
- mlx_result.stats.total_allocs, mlx_result.stats.total_frees,
- "MLX allocator: allocs should equal frees"
- );
- assert_eq!(
- heap_result.stats.total_allocs, heap_result.stats.total_frees,
- "Heap allocator: allocs should equal frees"
- );
-}
-
-#[test]
-fn test_long_conversation_comparison() {
- let num_conversations = 10;
- let avg_prompt_length = 512;
- let avg_generate_length = 1024;
- let prefill_step_size = 512;
- let snapshot_interval = 100;
-
- println!("\n========================================");
- println!(" Long Conversation Test");
- println!(
- " {} conversations, ~{}p + ~{}g tokens each",
- num_conversations, avg_prompt_length, avg_generate_length
- );
- println!("========================================");
-
- let gc_limit = 256 * 1024 * 1024;
- let max_pool = 128 * 1024 * 1024;
-
- let uzu_allocator = CachingAllocator::new();
- uzu_allocator.context.allocator().set_gc_limit(gc_limit);
- uzu_allocator.context.allocator().set_max_pool_size(max_pool);
-
- let uzu_result = run_long_simulation(
- &uzu_allocator,
- num_conversations,
- avg_prompt_length,
- avg_generate_length,
- prefill_step_size,
- snapshot_interval,
- );
-
- let mlx_allocator = MlxAllocator::new(gc_limit, max_pool);
- let mlx_result = run_long_simulation(
- &mlx_allocator,
- num_conversations,
- avg_prompt_length,
- avg_generate_length,
- prefill_step_size,
- snapshot_interval,
- );
-
- let heap_allocator = MTLHeapAllocator::new(1024 * 1024 * 1024);
- let heap_result = run_long_simulation(
- &heap_allocator,
- num_conversations,
- avg_prompt_length,
- avg_generate_length,
- prefill_step_size,
- snapshot_interval,
- );
-
- print_three_way_comparison(
- "Long Conversation Results",
- &uzu_result,
- &mlx_result,
- &heap_result,
- );
-
- print_memory_over_time("UZU Caching", &uzu_result.snapshots, 15);
- print_memory_over_time("MLX-style LRU", &mlx_result.snapshots, 15);
- print_memory_over_time("Heap (baseline)", &heap_result.snapshots, 15);
-
- println!("\n--- Memory Efficiency Summary ---");
- println!(
- "UZU peak/requested ratio: {:.2}%",
- (uzu_result.peak_memory_bytes as f64
- / uzu_result.stats.total_bytes_requested as f64)
- * 100.0
- );
- println!(
- "MLX peak/requested ratio: {:.2}%",
- (mlx_result.peak_memory_bytes as f64
- / mlx_result.stats.total_bytes_requested as f64)
- * 100.0
- );
- println!(
- "Heap peak/requested ratio: {:.2}%",
- (heap_result.peak_memory_bytes as f64
- / heap_result.stats.total_bytes_requested as f64)
- * 100.0
- );
-
- assert_eq!(
- uzu_result.stats.total_allocs, uzu_result.stats.total_frees,
- "UZU allocator: allocs should equal frees"
- );
- assert_eq!(
- mlx_result.stats.total_allocs, mlx_result.stats.total_frees,
- "MLX allocator: allocs should equal frees"
- );
-}
diff --git a/crates/uzu/tests/allocator_test/model_config.rs b/crates/uzu/tests/allocator_test/model_config.rs
deleted file mode 100644
index f398da479..000000000
--- a/crates/uzu/tests/allocator_test/model_config.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-#[derive(Clone, Copy)]
-pub struct ModelConfig {
- pub name: &'static str,
- pub model_dim: usize,
- pub num_heads: usize,
- pub num_groups: usize,
- pub head_dim: usize,
- pub hidden_dim: usize,
- pub vocab_size: usize,
- pub num_layers: usize,
-}
-
-impl ModelConfig {
- pub fn llama_3_2_1b() -> Self {
- Self {
- name: "Llama-3.2-1B",
- model_dim: 2048,
- num_heads: 32,
- num_groups: 8,
- head_dim: 64,
- hidden_dim: 8192,
- vocab_size: 128256,
- num_layers: 16,
- }
- }
-
- pub fn llama_3_2_3b() -> Self {
- Self {
- name: "Llama-3.2-3B",
- model_dim: 3072,
- num_heads: 24,
- num_groups: 8,
- head_dim: 128,
- hidden_dim: 8192,
- vocab_size: 128256,
- num_layers: 28,
- }
- }
-
- pub fn qwen3_4b() -> Self {
- Self {
- name: "Qwen3-4B",
- model_dim: 2560,
- num_heads: 32,
- num_groups: 8,
- head_dim: 128,
- hidden_dim: 9728,
- vocab_size: 151936,
- num_layers: 36,
- }
- }
-
- pub fn qwen3_8b() -> Self {
- Self {
- name: "Qwen3-8B",
- model_dim: 4096,
- num_heads: 32,
- num_groups: 8,
- head_dim: 128,
- hidden_dim: 12288,
- vocab_size: 151936,
- num_layers: 36,
- }
- }
-
- pub fn gemma_3_4b() -> Self {
- Self {
- name: "Gemma-3-4B",
- model_dim: 2560,
- num_heads: 8,
- num_groups: 4,
- head_dim: 256,
- hidden_dim: 10240,
- vocab_size: 262208,
- num_layers: 34,
- }
- }
-
- pub fn qwen3_14b() -> Self {
- Self {
- name: "Qwen3-14B",
- model_dim: 5120,
- num_heads: 40,
- num_groups: 8,
- head_dim: 128,
- hidden_dim: 17408,
- vocab_size: 151936,
- num_layers: 40,
- }
- }
-
- pub fn all_models() -> Vec<Self> {
- vec![
- Self::llama_3_2_1b(),
- Self::llama_3_2_3b(),
- Self::qwen3_4b(),
- Self::qwen3_8b(),
- Self::gemma_3_4b(),
- Self::qwen3_14b(),
- ]
- }
-
- pub fn qkv_dim(&self) -> usize {
- (2 * self.num_groups + self.num_heads) * self.head_dim
- }
-}
diff --git a/crates/uzu/tests/allocator_test/simulation_result.rs b/crates/uzu/tests/allocator_test/simulation_result.rs
deleted file mode 100644
index d3f2a3184..000000000
--- a/crates/uzu/tests/allocator_test/simulation_result.rs
+++ /dev/null
@@ -1,113 +0,0 @@
-use super::{
- allocation_stats::AllocationStats,
- allocator_trait::AllocatorTrait,
- inference_simulation::{InferenceSimulation, MemorySnapshot},
- model_config::ModelConfig,
-};
-
-pub struct SimulationResult {
- pub stats: AllocationStats,
- pub peak_memory_bytes: usize,
- pub cache_memory_bytes: usize,
- #[allow(dead_code)]
- pub final_prefix_len: usize,
- pub snapshots: Vec<MemorySnapshot>,
-}
-
-pub fn run_simulation<A: AllocatorTrait>(
- allocator: &A,
- prompt_length: usize,
- generate_length: usize,
- prefill_step_size: usize,
-) -> SimulationResult {
- let config = ModelConfig::llama_3_2_1b();
- let mut sim = InferenceSimulation::new(allocator, config);
-
- let mut snapshots = sim.run_prefill(prompt_length, prefill_step_size);
- let gen_snapshots = sim.run_generation(generate_length);
- snapshots.extend(gen_snapshots);
-
- SimulationResult {
- stats: sim.stats,
- peak_memory_bytes: allocator.peak_memory(),
- cache_memory_bytes: allocator.cache_memory(),
- final_prefix_len: sim.current_prefix_len,
- snapshots,
- }
-}
-
-pub fn run_long_simulation<A: AllocatorTrait>(
- allocator: &A,
- num_conversations: usize,
- avg_prompt_length: usize,
- avg_generate_length: usize,
- prefill_step_size: usize,
- snapshot_interval: usize,
-) -> SimulationResult {
- let config = ModelConfig::llama_3_2_1b();
- run_long_simulation_with_config(
- allocator,
- config,
- num_conversations,
- avg_prompt_length,
- avg_generate_length,
- prefill_step_size,
- snapshot_interval,
- )
-}
-
-pub fn run_simulation_with_config<A: AllocatorTrait>(
- allocator: &A,
- config: ModelConfig,
- prompt_length: usize,
- generate_length: usize,
- prefill_step_size: usize,
-) -> SimulationResult {
- let mut sim = InferenceSimulation::new(allocator, config);
-
- let mut snapshots = sim.run_prefill(prompt_length, prefill_step_size);
- let gen_snapshots = sim.run_generation(generate_length);
- snapshots.extend(gen_snapshots);
-
- SimulationResult {
- stats: sim.stats,
- peak_memory_bytes: allocator.peak_memory(),
- cache_memory_bytes: allocator.cache_memory(),
- final_prefix_len: sim.current_prefix_len,
- snapshots,
- }
-}
-
-pub fn run_long_simulation_with_config<A: AllocatorTrait>(
- allocator: &A,
- config: ModelConfig,
- num_conversations: usize,
- avg_prompt_length: usize,
- avg_generate_length: usize,
- prefill_step_size: usize,
- snapshot_interval: usize,
-) -> SimulationResult {
- let mut sim = InferenceSimulation::new(allocator, config);
- let mut all_snapshots = Vec::new();
-
- for conv_idx in 0..num_conversations {
- let prompt_len = avg_prompt_length + (conv_idx % 3) * (avg_prompt_length / 4);
- let gen_len = avg_generate_length + (conv_idx % 5) * (avg_generate_length / 8);
-
- let prefill_snapshots = sim.run_prefill(prompt_len, prefill_step_size);
- all_snapshots.extend(prefill_snapshots);
-
- let gen_snapshots = sim.run_generation_with_sampling(gen_len, snapshot_interval);
- all_snapshots.extend(gen_snapshots);
-
- sim.current_prefix_len = 0;
- }
-
- SimulationResult {
- stats: sim.stats,
- peak_memory_bytes: allocator.peak_memory(),
- cache_memory_bytes: allocator.cache_memory(),
- final_prefix_len: sim.current_prefix_len,
- snapshots: all_snapshots,
- }
-}
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index d53cf4c76..5e52c29db 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -5,14 +5,15 @@ mod common;
use std::mem::size_of;
use bytemuck;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use ndarray::{Array3, Array4, s};
use objc2::{rc::Retained, runtime::ProtocolObject};
use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBuffer, Context, Kernels,
+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending, Context, Kernels,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
@@ -316,8 +317,7 @@ fn run_single_pass_attention(
.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -347,12 +347,9 @@ fn run_single_pass_attention(
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
- &mut compute_encoder,
+ &mut command_buffer,
);
- compute_encoder.end_encoding();
-
- command_buffer.commit();
- command_buffer.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -389,8 +386,7 @@ fn run_single_pass_attention_with_is_causal(
.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -419,12 +415,9 @@ fn run_single_pass_attention_with_is_causal(
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
- &mut compute_encoder,
+ &mut command_buffer,
);
- compute_encoder.end_encoding();
-
- command_buffer.commit();
- command_buffer.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -460,8 +453,7 @@ fn run_gemm_attention(
.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let args = AttentionGemmArguments {
queries_buffer: &query_buffer,
@@ -481,12 +473,8 @@ fn run_gemm_attention(
scale,
};
- let encode_result = kernel.encode(context, &mut compute_encoder, args);
- compute_encoder.end_encoding();
- encode_result?;
-
- command_buffer.commit();
- command_buffer.wait_until_completed().unwrap();
+ kernel.encode(context, &mut command_buffer, args)?;
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -935,8 +923,7 @@ fn run_two_pass_attention(
)
.expect("Failed to create buffer");
- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -967,7 +954,7 @@ fn run_two_pass_attention(
mask_q_seq_stride,
mask_head_stride,
sinks_buffer.as_ref().map(|b| b),
- &mut compute_encoder,
+ &mut command_buffer,
);
kernel_pass2.encode(
&partials_buffer,
@@ -976,12 +963,9 @@ fn run_two_pass_attention(
&mut output_buffer,
num_heads as u32,
seq_len as u32,
- &mut compute_encoder,
+ &mut command_buffer,
);
- compute_encoder.end_encoding();
-
- command_buffer.commit();
- command_buffer.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -1173,8 +1157,7 @@ fn perf_two_pass_attention() {
.expect("Failed to create buffer");
// ---- Launch and time ----
- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
@@ -1199,7 +1182,7 @@ fn perf_two_pass_attention() {
None,
None,
sinks_buffer.as_ref().map(|b| b),
- &mut compute_encoder,
+ &mut command_buffer,
);
kernel_pass2.encode(
&partials_buffer,
@@ -1208,18 +1191,15 @@ fn perf_two_pass_attention() {
&mut output_buffer,
num_heads as u32,
suffix_length as u32,
- &mut compute_encoder,
+ &mut command_buffer,
);
- compute_encoder.end_encoding();
-
// Time both host-side and GPU execution
let host_timer = Instant::now();
- command_buffer.commit();
- command_buffer.wait_until_completed().unwrap();
+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
// Get actual GPU execution time
- let gpu_elapsed_ms = command_buffer.gpu_execution_time_ms();
+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();
match gpu_elapsed_ms {
Some(gpu_time) => {
diff --git a/crates/uzu/tests/kernel/full_precision_embedding_test.rs b/crates/uzu/tests/kernel/full_precision_embedding_test.rs
index ee634ba65..2a1b61fd8 100644
--- a/crates/uzu/tests/kernel/full_precision_embedding_test.rs
+++ b/crates/uzu/tests/kernel/full_precision_embedding_test.rs
@@ -8,7 +8,10 @@ use num_traits::Float;
use uzu::{
ArrayElement,
array::ArrayContextExt,
- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::FullPrecisionEmbeddingLookupKernel},
+ backends::common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Kernels, kernel::FullPrecisionEmbeddingLookupKernel,
+ },
};
struct Input<T: ArrayElement + Float> {
@@ -60,21 +63,18 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights, "");
let output_array = context.create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to get command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(
- token_ids_array.buffer().borrow().deref(),
- weights_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- input.batch_size as u32,
- input.vocab_size as u32,
- input.model_dim as u32,
- input.input_scale,
- encoder,
- );
- });
- command_buffer.submit();
- command_buffer.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to get command buffer").start_encoding();
+ kernel.encode(
+ token_ids_array.buffer().borrow().deref(),
+ weights_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size as u32,
+ input.vocab_size as u32,
+ input.model_dim as u32,
+ input.input_scale,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/kernel/qk_norm_test.rs b/crates/uzu/tests/kernel/qk_norm_test.rs
index 562f390fb..b587efecc 100644
--- a/crates/uzu/tests/kernel/qk_norm_test.rs
+++ b/crates/uzu/tests/kernel/qk_norm_test.rs
@@ -9,7 +9,10 @@ use uzu::{
ArrayElement, DataType,
array::ArrayContextExt,
backends::{
- common::{Backend, CommandBuffer, Context, Kernels, kernel::QKNormKernel},
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::QKNormKernel,
+ },
cpu::Cpu,
},
};
@@ -130,29 +133,24 @@ fn get_output<
let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");
let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(
- None::<&B::NativeBuffer>,
- scales_array.buffer().borrow().deref(),
- qkv_array.buffer().borrow_mut().deref_mut(),
- input.batch_size,
- input.num_q_heads,
- input.num_kv_heads,
- input.head_dim,
- input.epsilon,
- input.scale_offset,
- input.head_offset,
- input.head_count,
- input.full_layer,
- encoder,
- )
- });
-
- command_buffer.submit();
- if let Err(err) = command_buffer.wait_until_completed() {
- panic!("Failed to wait command buffer: {}", err);
- }
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ None::<&B::Buffer>,
+ scales_array.buffer().borrow().deref(),
+ qkv_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.num_q_heads,
+ input.num_kv_heads,
+ input.head_dim,
+ input.epsilon,
+ input.scale_offset,
+ input.head_offset,
+ input.head_count,
+ input.full_layer,
+ &mut command_buffer,
+ );
+
+ command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
qkv_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/kernel/rms_norm_test.rs b/crates/uzu/tests/kernel/rms_norm_test.rs
index dd6df922a..3b13ab441 100644
--- a/crates/uzu/tests/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/kernel/rms_norm_test.rs
@@ -13,7 +13,10 @@ use uzu::{
ArrayElement, DataType,
array::ArrayContextExt,
backends::{
- common::{Backend, CommandBuffer, Context, Kernels, kernel::RMSNormKernel},
+ common::{
+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending, Context, Kernels, kernel::RMSNormKernel,
+ },
cpu::Cpu,
},
};
@@ -145,28 +148,24 @@ fn get_output<
false => context.create_array_uninitialized(&[input_size], OutputT::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(
- input_buffer,
- scales_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- input.batch_size,
- input.element_count,
- input.epsilon,
- input.scale_offset,
- input.full_layer,
- encoder,
- )
- });
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ input_buffer,
+ scales_array.buffer().borrow().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.batch_size,
+ input.element_count,
+ input.epsilon,
+ input.scale_offset,
+ input.full_layer,
+ &mut command_buffer,
+ );
let instant = Instant::now();
- command_buffer.submit();
- if let Err(err) = command_buffer.wait_until_completed() {
- panic!("Failed to wait command buffer: {}", err);
- }
+ let completed =
+ command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let host_elapsed_ms = instant.elapsed().as_secs_f64() * 1e3;
- let gpu_elapsed_ms = command_buffer.gpu_execution_time_ms();
+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();
(output_array.as_slice().to_vec(), host_elapsed_ms, gpu_elapsed_ms)
}
diff --git a/crates/uzu/tests/kernel/tensor_add_bias_test.rs b/crates/uzu/tests/kernel/tensor_add_bias_test.rs
index 7014f6db1..363849284 100644
--- a/crates/uzu/tests/kernel/tensor_add_bias_test.rs
+++ b/crates/uzu/tests/kernel/tensor_add_bias_test.rs
@@ -8,7 +8,10 @@ use num_traits::Float;
use uzu::{
ArrayElement,
array::ArrayContextExt,
- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddBiasKernel},
+ backends::common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Kernels, kernel::TensorAddBiasKernel,
+ },
};
struct Input<T: ArrayElement + Float> {
@@ -75,19 +78,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
false => context.create_array_uninitialized(&[size], T::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(
- input_buffer,
- bias_array.buffer().borrow_mut().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- input.num_cols,
- input.length,
- encoder,
- )
- });
- command_buffer.submit();
- command_buffer.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ input_buffer,
+ bias_array.buffer().borrow_mut().deref(),
+ output_array.buffer().borrow_mut().deref_mut(),
+ input.num_cols,
+ input.length,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/kernel/tensor_add_swap_test.rs b/crates/uzu/tests/kernel/tensor_add_swap_test.rs
index d6c8b1952..cd5926cc8 100644
--- a/crates/uzu/tests/kernel/tensor_add_swap_test.rs
+++ b/crates/uzu/tests/kernel/tensor_add_swap_test.rs
@@ -5,7 +5,10 @@ use num_traits::Float;
use uzu::{
ArrayElement,
array::ArrayContextExt,
- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorAddSwapKernel},
+ backends::common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Kernels, kernel::TensorAddSwapKernel,
+ },
};
struct Input<T: ArrayElement + Float> {
@@ -46,17 +49,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");
let main_array = context.create_array_from(&[size], &input.main_buffer, "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(
- skip_array.buffer().borrow_mut().deref_mut(),
- main_array.buffer().borrow_mut().deref_mut(),
- input.length,
- encoder,
- )
- });
- command_buffer.submit();
- command_buffer.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ skip_array.buffer().borrow_mut().deref_mut(),
+ main_array.buffer().borrow_mut().deref_mut(),
+ input.length,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
(skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())
}
diff --git a/crates/uzu/tests/kernel/tensor_copy_test.rs b/crates/uzu/tests/kernel/tensor_copy_test.rs
index 4870584f9..e56ceef41 100644
--- a/crates/uzu/tests/kernel/tensor_copy_test.rs
+++ b/crates/uzu/tests/kernel/tensor_copy_test.rs
@@ -8,7 +8,10 @@ use num_traits::Float;
use uzu::{
ArrayElement,
array::ArrayContextExt,
- backends::common::{Backend, CommandBuffer, Context, Kernels, kernel::TensorCopyKernel},
+ backends::common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Kernels, kernel::TensorCopyKernel,
+ },
};
struct Input<T: ArrayElement + Float> {
@@ -45,17 +48,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let src_array = context.create_array_from(&[size], &input.src, "");
let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(
- src_array.buffer().borrow().deref(),
- dst_array.buffer().borrow_mut().deref_mut(),
- input.length,
- encoder,
- )
- });
- command_buffer.submit();
- command_buffer.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ src_array.buffer().borrow().deref(),
+ dst_array.buffer().borrow_mut().deref_mut(),
+ input.length,
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
dst_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/kernel/token_copy_test.rs b/crates/uzu/tests/kernel/token_copy_test.rs
index fa47b5e77..78b2c95b9 100644
--- a/crates/uzu/tests/kernel/token_copy_test.rs
+++ b/crates/uzu/tests/kernel/token_copy_test.rs
@@ -4,7 +4,8 @@ use uzu::{
ArrayElement,
array::ArrayContextExt,
backends::common::{
- Backend, CommandBuffer, Context, Kernels,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Kernels,
kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},
},
};
@@ -18,12 +19,13 @@ fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {
let src_array = context.create_array_from(&[1], &[src_value], "");
let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)
- });
- command_buffer.submit();
- command_buffer.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ src_array.buffer().borrow().deref(),
+ dst_array.buffer().borrow_mut().deref_mut(),
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let output = dst_array.as_slice::<u64>().to_vec();
assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());
@@ -38,12 +40,13 @@ fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {
let src_array = context.create_array_from(&[1], &[src_value], "");
let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
- command_buffer.with_compute_encoder(|encoder| {
- kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), encoder)
- });
- command_buffer.submit();
- command_buffer.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(
+ src_array.buffer().borrow().deref(),
+ dst_array.buffer().borrow_mut().deref_mut(),
+ &mut command_buffer,
+ );
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let output = dst_array.as_slice::<u32>().to_vec();
assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index 4573536af..e73dc063b 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -1,12 +1,13 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
-use metal::{MTLCommandBuffer, MTLCommandQueue};
-use objc2::Message;
use uzu::{
DataType,
array::ArrayContextExt,
backends::{
- common::{Backend, Context, kernel::kv_cache_update::KVCacheUpdate},
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, kernel::kv_cache_update::KVCacheUpdate,
+ },
metal::Metal,
},
forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
@@ -185,18 +186,15 @@ fn run_scenario(
},
};
- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
-
- let root_command_buffer = command_buffer.clone();
+ let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();
layer.update_after_acceptance(
&scenario.accepted_suffix_indices,
scenario.suffix_start,
- &mut root_command_buffer.retain(),
+ &mut command_buffer,
&kv_cache_update,
);
- command_buffer.commit();
- root_command_buffer.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
layer.register_accepted_tokens(&scenario.accepted_token_positions);
diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rs
index 90b93f9d6..da5c15980 100644
--- a/crates/uzu/tests/kv_cache_update_test.rs
+++ b/crates/uzu/tests/kv_cache_update_test.rs
@@ -3,11 +3,11 @@
use std::{cell::RefCell, rc::Rc};
use bytemuck;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use ndarray::{Array, Array3, s};
use uzu::backends::{
common::{
- Backend, Context,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
gpu_types::Swap,
kernel::kv_cache_update::{KVCacheUpdate, KVLayerData, create_swaps_direct},
},
@@ -106,8 +106,7 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
value_shape: [num_heads, seq_len, head_dim],
};
- let mut command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();
match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut command_buffer) {
Ok(_) => {},
Err(e) => {
@@ -116,8 +115,7 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
},
}
- command_buffer.commit();
- command_buffer.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;
let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 389f08496..0e1723233 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -2,13 +2,14 @@
use bytemuck;
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use ndarray::Array2;
use uzu::{
DataType,
backends::{
common::{
- Backend, Context,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context,
kernel::matmul::{MatmulArguments, MatmulKernel},
},
metal::{Metal, choose_dispatch_descriptor},
@@ -45,8 +46,7 @@ fn run_metal_matmul(
let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
let mut arguments = MatmulArguments {
a: &a_buf,
a_offset: 0,
@@ -64,11 +64,8 @@ fn run_metal_matmul(
};
MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);
let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");
- let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut enc);
- enc.end_encoding();
- encode_result.expect("encode");
- cb.commit();
- cb.wait_until_completed();
+ kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut command_buffer).expect("encode");
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
unsafe {
let ptr = d_buf.contents().as_ptr() as *const bf16;
diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
index 6d1164f1f..e79359560 100644
--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs
+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
@@ -1,14 +1,15 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
+use metal::MTLBuffer;
use objc2::{rc::Retained, runtime::ProtocolObject};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
common::{
- Backend, Kernels,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels,
kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},
},
metal::Metal,
@@ -75,8 +76,7 @@ fn gen_topk_ids_from_logits(
let mut topk_probs_buf = alloc_buffer::<bf16>(ctx, t * k);
// Use fused router+topk kernel
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16)
.expect("router_topk kernel");
router_topk.encode(
@@ -90,11 +90,9 @@ fn gen_topk_ids_from_logits(
e as u32,
k as u32,
true,
- &mut router_topk_encoder,
+ &mut command_buffer,
);
- router_topk_encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let ids_ptr = topk_ids_buf.contents().as_ptr() as *const i32;
let ids = unsafe { std::slice::from_raw_parts(ids_ptr, t * k) };
@@ -126,8 +124,8 @@ fn test_counts_offsets_fused_parity_random() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer =
+ ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel.encode(
&topk_ids_buf,
@@ -137,12 +135,10 @@ fn test_counts_offsets_fused_parity_random() {
t as u32,
e as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Verify offsets
let offsets_ptr = offsets_buf.contents().as_ptr() as *const u32;
@@ -178,8 +174,7 @@ fn test_counts_offsets_fused_edge_cases() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -188,11 +183,9 @@ fn test_counts_offsets_fused_edge_cases() {
t as u32,
e as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };
@@ -211,8 +204,7 @@ fn test_counts_offsets_fused_edge_cases() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -221,11 +213,9 @@ fn test_counts_offsets_fused_edge_cases() {
t as u32,
e as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };
assert!(offsets_gpu.iter().all(|&v| v == 0));
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index 4359b624a..a30ea799f 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -1,11 +1,12 @@
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
+use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
common::{
- Backend, Kernels,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
@@ -346,12 +347,11 @@ fn run_moe_parity_test_internal(
// Encode ALL kernels in one command buffer
eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
// Router + TopK (fused kernel)
let router_topk =
<<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk");
- let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -363,13 +363,11 @@ fn run_moe_parity_test_internal(
e as u32,
k as u32,
true,
- &mut router_topk_encoder,
+ &mut command_buffer,
);
- router_topk_encoder.end_encoding();
let fused_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
fused_kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -378,11 +376,9 @@ fn run_moe_parity_test_internal(
t as u32,
e as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)
.expect("scatter bases kernel");
scatter_bases_kernel.encode(
@@ -393,11 +389,9 @@ fn run_moe_parity_test_internal(
num_blocks as u32,
num_tiles as u32,
0u32,
- &mut scatter_bases_encoder,
+ &mut command_buffer,
);
- scatter_bases_encoder.end_encoding();
- let mut scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");
let scatter_map_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");
@@ -415,13 +409,12 @@ fn run_moe_parity_test_internal(
num_blocks as u32,
num_tiles as u32,
&mut tok2row_buf,
- &mut scatter_encoder,
+ &mut command_buffer,
);
- scatter_encoder.end_encoding();
let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");
gather.encode(
- &mut cb,
+ &mut command_buffer,
DataType::BF16,
MoeGatherArguments {
x_buffer: &x_buf,
@@ -469,11 +462,10 @@ fn run_moe_parity_test_internal(
silu_alpha,
data_type: DataType::BF16,
};
- experts.encode(&mut cb, args);
+ experts.encode(&mut command_buffer, args);
let finalize =
<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
&tok2row_buf,
&topk_probs_buf,
@@ -482,13 +474,11 @@ fn run_moe_parity_test_internal(
t as u32,
d_model as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
eprintln!("[E2E] GPU execution completed");
// Read GPU output
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index af5e63c1e..8fd419e07 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -1,13 +1,13 @@
use std::time::Instant;
use half::bf16;
-use metal::{MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
common::{
- Backend,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context,
kernel::moe::{
MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
@@ -106,9 +106,9 @@ fn run_decode_case(
let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);
for _ in 0..warmup {
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
experts_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -138,16 +138,15 @@ fn run_decode_case(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
experts_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -177,8 +176,7 @@ fn run_decode_case(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -270,7 +268,7 @@ fn run_two_pass_prefill_case(
let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);
for _ in 0..warmup {
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -299,15 +297,14 @@ fn run_two_pass_prefill_case(
silu_alpha: 1.702,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut cb, args);
- cb.commit();
- cb.wait_until_completed();
+ experts_kernel.encode(&mut command_buffer, args);
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -336,9 +333,8 @@ fn run_two_pass_prefill_case(
silu_alpha: 1.702,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut cb, args);
- cb.commit();
- cb.wait_until_completed();
+ experts_kernel.encode(&mut command_buffer, args);
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -413,9 +409,9 @@ fn run_fused_single_token_case(
let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);
for _ in 0..warmup {
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
fused_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -438,16 +434,15 @@ fn run_fused_single_token_case(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
fused_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -470,8 +465,7 @@ fn run_fused_single_token_case(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -593,9 +587,9 @@ fn run_indirect_decode_timed(
let mut row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);
for _ in 0..warmup {
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
experts_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -625,16 +619,15 @@ fn run_indirect_decode_timed(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
experts_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -664,8 +657,7 @@ fn run_indirect_decode_timed(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -712,9 +704,9 @@ fn run_fused_decode_timed(
let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);
for _ in 0..warmup {
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
fused_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -737,16 +729,15 @@ fn run_fused_decode_timed(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
fused_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -769,8 +760,7 @@ fn run_fused_decode_timed(
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index 15634b047..0afb09350 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -9,14 +9,17 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
+use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::moe::{
- MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
- MoeExpertsTwoPassPrefillBlock,
+ common::{
+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ kernel::moe::{
+ MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
+ MoeExpertsTwoPassPrefillBlock,
+ },
},
metal::Metal,
},
@@ -416,13 +419,13 @@ fn test_two_pass_decode_correctness() {
// Execute 2-pass decode kernel
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
const K_TILE: usize = 64;
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;
experts_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -453,8 +456,7 @@ fn test_two_pass_decode_correctness() {
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Read GPU partial output and do CPU finalize (weighted sum)
let y_partial_gpu =
@@ -560,9 +562,9 @@ fn test_two_pass_decode_multi_token() {
let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
experts_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -592,8 +594,7 @@ fn test_two_pass_decode_multi_token() {
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let y_partial_gpu =
unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
@@ -662,7 +663,7 @@ fn test_two_pass_prefill_correctness() {
let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -691,9 +692,8 @@ fn test_two_pass_prefill_correctness() {
silu_alpha,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut cb, args);
- cb.commit();
- cb.wait_until_completed();
+ experts_kernel.encode(&mut command_buffer, args);
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let y_partial_gpu =
unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
@@ -791,10 +791,10 @@ fn test_fused_single_token_decode() {
// Run fused decode kernel
let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
fused_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -818,8 +818,7 @@ fn test_fused_single_token_decode() {
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Read GPU output
let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };
@@ -915,9 +914,9 @@ fn test_fused_single_token_k4() {
let mut y_buf = alloc_buffer::<bf16>(&ctx, d_model);
let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
fused_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -940,8 +939,7 @@ fn test_fused_single_token_k4() {
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };
diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rs
index e292058b0..3677d3a5c 100644
--- a/crates/uzu/tests/moe/moe_finalize_test.rs
+++ b/crates/uzu/tests/moe/moe_finalize_test.rs
@@ -1,12 +1,15 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
+use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::{Backend, Kernels, kernel::MoeFinalizeKernel},
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::MoeFinalizeKernel,
+ },
metal::Metal,
},
};
@@ -90,8 +93,7 @@ fn test_finalize_correctness() {
// Execute finalize kernel
let finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16)
.expect("finalize kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
finalize.encode(
&tok2row_buf,
&probs_buf,
@@ -100,11 +102,9 @@ fn test_finalize_correctness() {
t as u32,
d_model as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let y_gpu = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };
diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rs
index 7dfedae9b..f9cf5a01a 100644
--- a/crates/uzu/tests/moe/moe_gather_test.rs
+++ b/crates/uzu/tests/moe/moe_gather_test.rs
@@ -1,12 +1,15 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
+use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::kernel::moe::{MoeGatherArguments, MoeGatherKernels},
+ common::{
+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ kernel::moe::{MoeGatherArguments, MoeGatherKernels},
+ },
metal::Metal,
},
};
@@ -78,9 +81,9 @@ fn test_gather_correctness() {
// Execute gather kernel using kernel struct
let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
gather.encode(
- &mut cb,
+ &mut command_buffer,
DataType::BF16,
MoeGatherArguments {
x_buffer: &x_buf,
@@ -92,8 +95,7 @@ fn test_gather_correctness() {
d_model,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let x_gpu =
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index 7d2b61310..1e734200c 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -1,13 +1,13 @@
use std::time::Instant;
use half::bf16;
-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
common::{
- Backend, Kernels,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
@@ -121,8 +121,8 @@ fn test_moe_e2e_decode_perf() {
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer =
+ ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -134,11 +134,9 @@ fn test_moe_e2e_decode_perf() {
e as u32,
k as u32,
true,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
fused_perf.print();
@@ -182,8 +180,8 @@ fn test_moe_e2e_prefill_perf() {
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer =
+ ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -195,11 +193,9 @@ fn test_moe_e2e_prefill_perf() {
e as u32,
k as u32,
true,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
fused_perf.print();
@@ -311,8 +307,7 @@ fn test_moe_pipeline_breakdown_decode() {
// Testing: Router + TopK + Counts+Offsets (FUSED)
let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
router_topk_fused_kernel.encode(
&x_buf,
&router_w_buf,
@@ -324,16 +319,13 @@ fn test_moe_pipeline_breakdown_decode() {
e as u32,
k as u32,
true,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
counts_offsets_kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -342,16 +334,13 @@ fn test_moe_pipeline_breakdown_decode() {
t as u32,
e as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
let scatter_perf = time_kernel("Scatter", 2, 5, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
scatter_bases_kernel.encode(
&partials_buf,
&mut block_bases_buf,
@@ -360,11 +349,8 @@ fn test_moe_pipeline_breakdown_decode() {
num_blocks as u32,
num_tiles as u32,
0u32,
- &mut scatter_bases_encoder,
+ &mut command_buffer,
);
- scatter_bases_encoder.end_encoding();
-
- let mut scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");
scatter_map_kernel.encode(
&topk_ids_buf,
&topk_probs_buf,
@@ -379,18 +365,15 @@ fn test_moe_pipeline_breakdown_decode() {
num_blocks as u32,
num_tiles as u32,
&mut tok2row_buf,
- &mut scatter_map_encoder,
+ &mut command_buffer,
);
- scatter_map_encoder.end_encoding();
-
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
let gather_perf = time_kernel("Gather", 2, 5, || {
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
gather_kernel.encode(
- &mut cb,
+ &mut command_buffer,
DataType::BF16,
MoeGatherArguments {
x_buffer: &x_buf,
@@ -402,14 +385,13 @@ fn test_moe_pipeline_breakdown_decode() {
d_model,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
experts_kernel.encode(
- &mut cb,
+ &mut command_buffer,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -439,13 +421,11 @@ fn test_moe_pipeline_breakdown_decode() {
data_type: DataType::BF16,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
let finalize_perf = time_kernel("Finalize", 2, 5, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
finalize_kernel.encode(
&tok2row_buf,
&topk_probs_buf,
@@ -454,11 +434,9 @@ fn test_moe_pipeline_breakdown_decode() {
t as u32,
d_model as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
});
// Print results
diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rs
index 9fc90a90b..cb869b65b 100644
--- a/crates/uzu/tests/moe/moe_router_topk_test.rs
+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs
@@ -1,12 +1,15 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
+use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::{Backend, Kernels, kernel::MoeRouterTopKKernel},
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::MoeRouterTopKKernel,
+ },
metal::Metal,
},
};
@@ -157,8 +160,7 @@ fn run_router_topk_once(
// For BFloat16 kernel, probs buffer must be bf16, not f32
let mut probs_buf = alloc_buffer::<bf16>(ctx, t * k);
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("Failed to create command encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel.encode(
&input_buf,
&weight_buf,
@@ -170,11 +172,9 @@ fn run_router_topk_once(
e as u32,
k as u32,
renorm,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let ids_ptr = ids_buf.contents().as_ptr() as *const i32;
let probs_ptr = probs_buf.contents().as_ptr() as *const bf16;
diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rs
index 3faf73fc3..a00356825 100644
--- a/crates/uzu/tests/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/moe/moe_scatter_test.rs
@@ -1,13 +1,14 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
+use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
common::{
- Backend, Kernels,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,
MoeScatterBucketsKernel,
@@ -84,8 +85,7 @@ fn test_scatter_buckets_parity() {
// Use fused router+topk kernel
let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
.expect("router_topk");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
router_topk.encode(
&input_buf,
&weight_buf,
@@ -97,11 +97,9 @@ fn test_scatter_buckets_parity() {
e as u32,
k as u32,
true,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Read back CPU probs for reference compare
let probs_bf16 =
@@ -116,8 +114,7 @@ fn test_scatter_buckets_parity() {
let fused_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
fused_kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -126,11 +123,9 @@ fn test_scatter_buckets_parity() {
t as u32,
e as u32,
k as u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Partials already created by fused kernel above
let num_blocks = 1; // Fused kernel uses single block
@@ -143,11 +138,9 @@ fn test_scatter_buckets_parity() {
let scatter_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, DataType::BF16)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
let mut block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);
- let mut scatter_bases_encoder =
- cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");
scatter_bases_kernel.encode(
&partials_buf,
&mut block_bases_buf,
@@ -156,19 +149,16 @@ fn test_scatter_buckets_parity() {
num_blocks as u32,
num_tiles as u32,
0u32,
- &mut scatter_bases_encoder,
+ &mut command_buffer,
);
- scatter_bases_encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let sumk = unsafe { std::slice::from_raw_parts(sumk_buf.contents().as_ptr() as *const u32, 1) }[0] as usize;
let mut out_ids_buf = alloc_buffer::<i32>(&ctx, sumk);
let mut out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
scatter_kernel.encode(
&topk_ids_buf,
&topk_probs_buf,
@@ -182,12 +172,10 @@ fn test_scatter_buckets_parity() {
k as u32,
num_blocks as u32,
num_tiles as u32,
- &mut scatter_encoder,
+ &mut command_buffer,
);
- scatter_encoder.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let out_ids = unsafe { std::slice::from_raw_parts(out_ids_buf.contents().as_ptr() as *const i32, sumk) };
let out_probs_h = unsafe { std::slice::from_raw_parts(out_probs_buf.contents().as_ptr() as *const bf16, sumk) };
diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rs
index b0bd67301..9ab1ffc3e 100644
--- a/crates/uzu/tests/moe/moe_tiles_test.rs
+++ b/crates/uzu/tests/moe/moe_tiles_test.rs
@@ -1,9 +1,12 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
+use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::backends::{
- common::kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},
+ common::{
+ CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},
+ },
metal::Metal,
};
@@ -35,17 +38,16 @@ fn test_tile_counts_correctness() {
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
tile_kernel.encode_counts(
- &mut cb,
+ &mut command_buffer,
MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &mut tile_counts_buf,
e,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let tile_counts_gpu =
@@ -80,9 +82,9 @@ fn test_tile_scan_correctness() {
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
tile_kernel.encode_scan(
- &mut cb,
+ &mut command_buffer,
MoeTileScanArguments {
tile_counts_buffer: &tile_counts_buf,
tile_offsets_buffer: &mut tile_offsets_buf,
@@ -90,8 +92,7 @@ fn test_tile_scan_correctness() {
e,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let tile_offsets_gpu =
@@ -120,17 +121,16 @@ fn test_tile_edge_cases() {
let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
tile_kernel.encode_counts(
- &mut cb,
+ &mut command_buffer,
MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &mut tile_counts_buf,
e,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let tile_counts_gpu =
unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };
@@ -151,17 +151,16 @@ fn test_tile_edge_cases() {
let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
tile_kernel.encode_counts(
- &mut cb,
+ &mut command_buffer,
MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &mut tile_counts_buf,
e,
},
);
- cb.commit();
- cb.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let tile_counts_gpu =
unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index 070956736..acedd70ea 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -2,13 +2,14 @@ use std::time::Instant;
use bytemuck;
use half::{bf16, f16};
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use objc2::{rc::Retained, runtime::ProtocolObject};
use uzu::{
DataType,
backends::{
common::{
- Backend, Context,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context,
kernel::quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
QuantizedMatmulType,
@@ -510,12 +511,9 @@ fn execute_quantized_matmul(
output_dim,
quantization_type,
};
- let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&mut encoder, args).unwrap();
- encoder.end_encoding();
- cb_ref.commit();
- cb_ref.wait_until_completed();
+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
+ kernel.encode(&mut command_buffer, args).unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
}
@@ -533,12 +531,9 @@ fn execute_quantized_matmul(
output_dim,
quantization_type,
};
- let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&mut encoder, args).unwrap();
- encoder.end_encoding();
- cb_ref.commit();
- cb_ref.wait_until_completed();
+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
+ kernel.encode(&mut command_buffer, args).unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
}
let elapsed = start.elapsed();
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index 5650f6b69..dd2839838 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -1,13 +1,14 @@
mod common;
use bytemuck;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use rand::{RngExt, seq::SliceRandom};
// for Vec::shuffle
use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBuffer, Context, Kernels,
+ Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
+ CommandBufferPending, Context, Kernels,
kernel::{
GumbelKernel, MinPKernel, TemperatureKernel, TopKKernel, TopPKernel,
sampling::{ArgmaxStrategy, SamplingKernel},
@@ -107,13 +108,9 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
.new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
-
- // Run sampling
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buffer,
None,
0,
@@ -123,13 +120,10 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
SamplingMethod::Greedy,
batch_size,
vocab_size,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("Argmax sampling should succeed");
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
// Check results
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
@@ -238,11 +232,10 @@ fn test_topp_sampling_from_prob_exact_match(
.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let cb = cb_ref.to_owned();
- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buf,
Some(&seeds_buf),
0,
@@ -257,12 +250,10 @@ fn test_topp_sampling_from_prob_exact_match(
},
batch_size,
vocab_size,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("encode");
- compute_encoder.end_encoding();
- cb_ref.commit();
- cb_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let ptr = output_buf.contents().as_ptr() as *const u32;
let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };
@@ -361,12 +352,11 @@ fn test_topp_sampling_statistical_large() {
.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let cb = cb_ref.to_owned();
- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buf,
Some(&seeds_buf),
0,
@@ -381,13 +371,10 @@ fn test_topp_sampling_statistical_large() {
},
BATCH,
VOCAB,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("encode");
- compute_encoder.end_encoding();
-
- cb_ref.commit();
- cb_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let ptr = output_buf.contents().as_ptr() as *const u32;
let sample_ids = unsafe { std::slice::from_raw_parts(ptr, BATCH) };
@@ -463,12 +450,10 @@ fn perf_topp_128k_vocab() {
.expect("Failed to create buffer");
// ---- Launch once and time ----
- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let cb = cb_ref.to_owned();
- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buf,
Some(&seeds_buf),
0,
@@ -483,19 +468,17 @@ fn perf_topp_128k_vocab() {
},
BATCH,
VOCAB,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("encode");
- compute_encoder.end_encoding();
// Time both host-side and GPU execution
let host_timer = Instant::now();
- cb_ref.commit();
- cb_ref.wait_until_completed().unwrap();
+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
// Get actual GPU execution time
- let gpu_elapsed_ms = cb.gpu_execution_time_ms();
+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();
match gpu_elapsed_ms {
Some(gpu_time) => {
@@ -567,12 +550,10 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
.expect("Failed to create buffer");
// ---- Launch once and time ----
- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let cb = cb_ref.to_owned();
- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buf,
Some(&seeds_buf),
0,
@@ -582,19 +563,17 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
SamplingMethod::Greedy,
BATCH,
VOCAB,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("encode");
- compute_encoder.end_encoding();
// Time both host-side and GPU execution
let host_timer = Instant::now();
- cb_ref.commit();
- cb_ref.wait_until_completed().unwrap();
+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
// Get actual GPU execution time
- let gpu_elapsed_ms = cb.gpu_execution_time_ms();
+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();
match gpu_elapsed_ms {
Some(gpu_time) => {
@@ -686,13 +665,11 @@ fn test_categorical_sampling() {
.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
- let mut compute_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buffer,
Some(&seeds_buffer),
0,
@@ -707,13 +684,10 @@ fn test_categorical_sampling() {
},
batch_size,
vocab_size,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("Categorical sampling should succeed");
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };
@@ -828,13 +802,11 @@ fn test_categorical_sampling_statistical() {
.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
- let mut compute_encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buffer,
Some(&seeds_buffer),
0,
@@ -849,13 +821,10 @@ fn test_categorical_sampling_statistical() {
},
BATCH,
VOCAB,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("encode");
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
let results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH) };
@@ -935,12 +904,10 @@ fn perf_categorical_128k_vocab() {
.expect("Failed to create buffer");
// Launch and time
- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let cb = cb_ref.to_owned();
- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buf,
Some(&seeds_buf),
0,
@@ -955,17 +922,15 @@ fn perf_categorical_128k_vocab() {
},
BATCH,
VOCAB,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("encode");
- compute_encoder.end_encoding();
let host_timer = Instant::now();
- cb_ref.commit();
- cb_ref.wait_until_completed().unwrap();
+ let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- let gpu_elapsed_ms = cb.gpu_execution_time_ms();
+ let gpu_elapsed_ms = completed.gpu_execution_time_ms();
match gpu_elapsed_ms {
Some(gpu_time) => {
@@ -1023,22 +988,16 @@ fn test_temperature_gpu_cpu_match() {
.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
-
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel.encode(
Some(&logits_buffer),
&mut processed_buffer,
BATCH as u32,
VOCAB as u32,
TEMPERATURE,
- &mut compute_encoder,
+ &mut command_buffer,
);
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;
let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };
@@ -1096,15 +1055,9 @@ fn test_topk_gpu_cpu_match() {
.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
-
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut compute_encoder);
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut command_buffer);
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
@@ -1161,15 +1114,9 @@ fn test_topp_gpu_cpu_match() {
.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
-
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut compute_encoder);
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut command_buffer);
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
@@ -1231,15 +1178,9 @@ fn test_minp_gpu_cpu_match() {
.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
-
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut compute_encoder);
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut command_buffer);
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
@@ -1320,11 +1261,10 @@ fn test_minp_sampling_exact_match(
.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let cb = cb_ref.to_owned();
- let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer =
+ context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel
- .encode_with_encoder(
+ .encode(
&mut logits_buf,
Some(&seeds_buf),
0,
@@ -1339,12 +1279,10 @@ fn test_minp_sampling_exact_match(
},
batch_size,
vocab_size,
- &mut compute_encoder,
+ &mut command_buffer,
)
.expect("encode");
- compute_encoder.end_encoding();
- cb_ref.commit();
- cb_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let ptr = output_buf.contents().as_ptr() as *const u32;
let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };
@@ -1418,22 +1356,16 @@ fn test_gumbel_gpu_cpu_match() {
.new_buffer(BATCH * VOCAB * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let command_buffer = command_buffer_ref.to_owned();
-
- let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
kernel.encode(
Some(&logits_buffer),
&seeds_buffer,
&mut gumbel_logits_buffer,
BATCH as u32,
VOCAB as u32,
- &mut compute_encoder,
+ &mut command_buffer,
);
- compute_encoder.end_encoding();
-
- command_buffer_ref.commit();
- command_buffer_ref.wait_until_completed().unwrap();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let result_ptr = gumbel_logits_buffer.contents().as_ptr() as *const f32;
let all_results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH * VOCAB) };
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index 470a714f8..70ca79769 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -3,16 +3,14 @@
use std::mem::size_of;
use bytemuck;
-use metal::{
- MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt,
- MTLResourceOptions,
-};
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use objc2::runtime::ProtocolObject;
use uzu::{
DataType,
backends::{
common::{
- Backend, Context, Kernels,
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels,
kernel::{
Conv1dScanKernel,
ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
@@ -248,12 +246,9 @@ fn run_prefill_kernel_mode(
head_dim: fixture.head_dim,
};
- let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&mut encoder, args, mode);
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
+ kernel.encode(&mut command_buffer, args, mode);
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let y_vec = read_buffer(&y_buf, fixture.total_x);
let state_vec = read_buffer(&state_buf, fixture.total_state);
@@ -290,7 +285,8 @@ fn run_conv_scan_once(
device.new_buffer_with_data(bytemuck::cast_slice(w_data), STORAGE_MODE).expect("Failed to create buffer");
let b_buf =
device.new_buffer_with_data(bytemuck::cast_slice(b_data), STORAGE_MODE).expect("Failed to create buffer");
- let state_buf = device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
+ let mut state_buf =
+ device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer");
let scratch_buf = if use_scratch && tap_count > 0 {
Some(device.new_buffer(total_state * size_of::<f32>(), STORAGE_MODE).expect("Failed to create buffer"))
} else {
@@ -320,8 +316,7 @@ fn run_conv_scan_once(
write_buffer(&padded_buf, &host);
}
- let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
let mut b_out = y_buf.clone();
let mut c_out = y_buf.clone();
let mut state_out = scratch_buf.as_ref().unwrap_or(&state_buf).clone();
@@ -340,21 +335,17 @@ fn run_conv_scan_once(
channels as u32,
channels as u32,
0u32,
- &mut encoder,
+ &mut command_buffer,
);
- encoder.end_encoding();
if let Some(ref scratch) = scratch_buf {
let bytes = channels * tap_count * size_of::<f32>();
if bytes > 0 {
- let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit encoder");
- blit.copy_buffer_to_buffer(scratch, 0, &state_buf, 0, bytes);
- blit.end_encoding();
+ command_buffer.encode_copy(scratch, &mut state_buf, bytes);
}
}
- command_buffer.commit();
- command_buffer.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let y_vec = read_buffer(&y_buf, total_x);
let state_vec = read_buffer(&state_buf, total_state);
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index af3377d2f..9567e0434 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -2,11 +2,14 @@
use bytemuck;
use half::bf16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
use uzu::{
DataType,
backends::{
- common::{Backend, Context, Kernels, kernel::SSDUpdateKernel},
+ common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context, Kernels, kernel::SSDUpdateKernel,
+ },
metal::Metal,
},
};
@@ -239,9 +242,7 @@ fn ssd_update_with_z_bf16() {
.expect("Failed to create buffer");
let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16, false).unwrap();
- let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let cb = cb_ref.to_owned();
- let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
kernel.encode(
&x_buf,
&dt_buf,
@@ -261,11 +262,9 @@ fn ssd_update_with_z_bf16() {
bsz as u32,
h as u32,
dh as u32,
- &mut enc,
+ &mut command_buffer,
);
- enc.end_encoding();
- cb_ref.commit();
- cb_ref.wait_until_completed();
+ command_buffer.end_encoding().submit().wait_until_completed().unwrap();
let y_ptr = y_buf.contents().as_ptr() as *const bf16;
let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };