Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

basic hazard tracker and memory allocator (#269)

uuuvn33fa7019bde6
diff --git a/Cargo.lock b/Cargo.lock
index 99d5be237..20b9c3509 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2956,6 +2956,12 @@ dependencies = [
"rand_core 0.9.5",
]
+[[package]]
+name = "rangemap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
+
[[package]]
name = "rawpointer"
version = "0.2.1"
@@ -4361,6 +4367,7 @@ dependencies = [
"quote",
"rand 0.10.0",
"rand_distr",
+ "rangemap",
"regex",
"schemars",
"serde",
diff --git a/Cargo.toml b/Cargo.toml
index e754234c3..d09f1bb1b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -50,6 +50,7 @@ walkdir = "2.5"
depfile = "0.1"
tempfile = "3.24"
clang-ast = "0.1"
+rangemap = "1.7.1"
futures = "0.3"
tokio = { version = "1.49", features = ["full"] }
blake3 = "1.8"
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index a2da1559f..791bfc9dc 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -13,6 +13,7 @@ serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tokenizers.workspace = true
+rangemap.workspace = true
bytemuck.workspace = true
zerocopy = { workspace = true, features = ["derive"] }
xxhash-rust = { workspace = true, features = ["xxh3"] }
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rs
index 6b0e8fd4a..1c110d652 100644
--- a/crates/uzu/build/common/traitgen.rs
+++ b/crates/uzu/build/common/traitgen.rs
@@ -76,7 +76,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
fn new(context: &<Self::Backend as crate::backends::common::Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as crate::backends::common::Backend>::Error>;
- fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#args ,)* command_buffer: &'command_buffer mut <<Self::Backend as crate::backends::common::Backend>::CommandBuffer as crate::backends::common::CommandBuffer>::Encoding);
+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder mut crate::backends::common::Encoder<Self::Backend>);
}
};
@@ -123,25 +123,53 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>
}
}
+ impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::Buffer> for &'a crate::backends::common::Allocation<B> {
+ fn into_parts(self) -> (&'a B::Buffer, usize) {
+ let (buffer, range) = self.as_buffer_range();
+ (buffer, range.start)
+ }
+ }
+
impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for (&'a B, usize) {
fn into_parts(self) -> (&'a B, usize) {
self
}
}
+ impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::Buffer> for (&'a crate::backends::common::Allocation<B>, usize) {
+ fn into_parts(self) -> (&'a B::Buffer, usize) {
+ let (buffer, range) = self.0.as_buffer_range();
+ (buffer, range.start + self.1)
+ }
+ }
+
pub trait BufferArgMut<'a, B: crate::backends::common::Buffer> {
- fn into_parts(self) -> (&'a mut B, usize);
+ fn into_parts(self) -> (&'a B, usize);
}
impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for &'a mut B {
- fn into_parts(self) -> (&'a mut B, usize) {
+ fn into_parts(self) -> (&'a B, usize) {
(self, 0)
}
}
+ impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::Buffer> for &'a mut crate::backends::common::Allocation<B> {
+ fn into_parts(self) -> (&'a B::Buffer, usize) {
+ let (buffer, range) = self.as_buffer_range();
+ (buffer, range.start)
+ }
+ }
+
impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for (&'a mut B, usize) {
- fn into_parts(self) -> (&'a mut B, usize) {
- self
+ fn into_parts(self) -> (&'a B, usize) {
+ (&*self.0, self.1)
+ }
+ }
+
+ impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::Buffer> for (&'a mut crate::backends::common::Allocation<B>, usize) {
+ fn into_parts(self) -> (&'a B::Buffer, usize) {
+ let (buffer, range) = self.0.as_buffer_range();
+ (buffer, range.start + self.1)
}
}
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rs
index da8c99615..4fd6b0555 100644
--- a/crates/uzu/build/cpu/compiler.rs
+++ b/crates/uzu/build/cpu/compiler.rs
@@ -340,8 +340,8 @@ impl CpuCompiler {
(
Some(quote! { #buffer_lifetime }),
match access {
- KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, Box<[u8]>> },
- KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, Box<[u8]>> },
+ KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },
+ KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },
},
)
},
@@ -371,9 +371,9 @@ impl CpuCompiler {
let argument_ident = &argument.name;
match &argument.ty {
FunctionArgumentType::Buffer(access) => {
- let as_ptr_fn = match access {
- KernelBufferAccess::Read => quote! { as_ptr },
- KernelBufferAccess::ReadWrite => quote! { as_mut_ptr },
+ let buffer_ptr = match access {
+ KernelBufferAccess::Read => quote! { (&*__dsl_buffer.get()).as_ptr() },
+ KernelBufferAccess::ReadWrite => quote! { (&mut *__dsl_buffer.get()).as_mut_ptr() },
};
if argument.conditional.is_some() {
@@ -381,7 +381,7 @@ impl CpuCompiler {
let #argument_ident = #argument_ident.map(|__dsl_buffer_impl| unsafe {
let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();
- __dsl_buffer.#as_ptr_fn().byte_add(__dsl_offset)
+ #buffer_ptr.byte_add(__dsl_offset)
});
})
} else {
@@ -389,7 +389,7 @@ impl CpuCompiler {
let #argument_ident = unsafe {
let (__dsl_buffer, __dsl_offset) = #argument_ident.into_parts();
- __dsl_buffer.#as_ptr_fn().byte_add(__dsl_offset)
+ #buffer_ptr.byte_add(__dsl_offset)
};
})
}
@@ -439,7 +439,7 @@ impl CpuCompiler {
});
quote! {
- encoder.push_command(move || #monomorphized_function(#function_call_args_joined));
+ encoder.as_command_buffer_mut().push_command(move || #monomorphized_function(#function_call_args_joined));
}
};
@@ -530,7 +530,7 @@ impl CpuCompiler {
})
}
- fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer) {
+ fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::common::Encoder<crate::backends::cpu::Cpu>) {
#(#argument_copies)*
#encode_body
}
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index 528554320..ecf017cfa 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -99,11 +99,13 @@ pub fn bindgen(
let mut arg_count: usize = 0;
let mut indirect_flag = false;
- let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_args_sets): (
+ let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_deconstructs, encode_accesses, encode_args_sets): (
Vec<Option<TokenStream>>,
Vec<Option<TokenStream>>,
Vec<Option<TokenStream>>,
Vec<TokenStream>,
+ Vec<Option<TokenStream>>,
+ Vec<Option<TokenStream>>,
Vec<TokenStream>,
) = kernel
.arguments
@@ -113,7 +115,7 @@ pub fn bindgen(
match ka.argument_type().unwrap() {
arg_type @ (MetalArgumentType::Buffer(_) | MetalArgumentType::Constant(_)) => {
- let (mut ty, mut set, generic) = match arg_type {
+ let (mut ty, deconstruct, access_type, mut set, generic) = match arg_type {
MetalArgumentType::Buffer(access) => {
let buffer_lifetime = Lifetime::new(&format!("'{}", ka.name.as_ref()), Span::call_site());
(
@@ -121,9 +123,18 @@ pub fn bindgen(
MetalBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, Retained<ProtocolObject<dyn MTLBuffer>>> },
MetalBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, Retained<ProtocolObject<dyn MTLBuffer>>> },
},
+ Some(if ka.argument_condition().unwrap().is_some() {
+ quote! {
+ let #arg_name = #arg_name.map(|#arg_name| #arg_name.into_parts());
+ }
+ } else {
+ quote! {
+ let #arg_name = #arg_name.into_parts();
+ }
+ }),
+ Some(access),
quote! {
- let (__dsl_buffer, __dsl_offset) = #arg_name.into_parts();
- compute_encoder.set_buffer(Some(__dsl_buffer), __dsl_offset, #arg_count);
+ compute_encoder.set_buffer(Some(#arg_name.0), #arg_name.1, #arg_count);
},
Some(quote! { #buffer_lifetime }),
)
@@ -133,11 +144,15 @@ pub fn bindgen(
match constant_type {
MetalConstantType::Scalar => (
quote! { #arg_dtype },
+ None,
+ None,
quote! { compute_encoder.set_value(&#arg_name, #arg_count); },
None,
),
MetalConstantType::Array => (
quote! { &[#arg_dtype] },
+ None,
+ None,
quote! { compute_encoder.set_slice(#arg_name, #arg_count); },
None,
),
@@ -146,12 +161,29 @@ pub fn bindgen(
_ => unreachable!(),
};
+ let mut access = access_type.map(|access| {
+ let compute_write = matches!(access, MetalBufferAccess::ReadWrite);
+
+ quote! {
+ crate::backends::common::Access {
+ range: #arg_name.0.gpu_address_subrange((#arg_name.1)..(#arg_name.0.length())),
+ flags: crate::backends::common::AccessFlags {
+ compute_read: true,
+ compute_write: #compute_write,
+ copy_read: false,
+ copy_write: false,
+ },
+ }
+ }
+ });
+
let (conditional_buffer_field, conditional_buffer_set) =
if let Some(condition) = ka.argument_condition().unwrap() {
let conditional_field_name = format_ident!("has_{}", ka.name.as_ref());
let condition = parse_expr(condition.as_ref()).unwrap();
ty = quote! { Option<#ty> };
+ access = access.map(|access| quote! { #arg_name.as_ref().map(|#arg_name| #access)});
set = quote! {
assert!(#arg_name.is_some() == (self.#conditional_field_name));
if let Some(#arg_name) = #arg_name {
@@ -164,6 +196,8 @@ pub fn bindgen(
Some(quote! { #conditional_field_name: #condition }),
)
} else {
+ access = access.map(|access| quote! { Some(#access)});
+
(None, None)
};
@@ -174,13 +208,28 @@ pub fn bindgen(
conditional_buffer_set,
generic,
quote! { #arg_name: #ty },
+ deconstruct,
+ access,
set,
))
}
MetalArgumentType::Groups(MetalGroupsType::Indirect) if !indirect_flag => {
indirect_flag = true;
- Some((None, None, Some(quote! { '__dsl_indirect_dispatch_buffer }), quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> }, quote! {}))
+ Some((
+ None,
+ None,
+ Some(quote! { '__dsl_indirect_dispatch_buffer }),
+ quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> },
+ Some(quote! { let __dsl_indirect_dispatch_buffer = __dsl_indirect_dispatch_buffer.into_parts(); }),
+ Some(quote! {
+ Some(crate::backends::common::Access {
+ range: __dsl_indirect_dispatch_buffer.0.gpu_address_subrange((__dsl_indirect_dispatch_buffer.1)..(__dsl_indirect_dispatch_buffer.1+12)),
+ flags: crate::backends::common::AccessFlags::compute_read(),
+ })
+ }),
+ quote! {},
+ ))
}
_ => None,
}
@@ -191,6 +240,22 @@ pub fn bindgen(
let conditional_buffer_sets = conditional_buffer_sets.into_iter().flatten().collect::<Vec<_>>();
let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();
+ let encode_deconstructs = encode_deconstructs.into_iter().flatten().collect::<Vec<_>>();
+
+ let encode_accesses = encode_accesses.into_iter().flatten().collect::<Vec<_>>();
+ let encode_accesses = if !encode_accesses.is_empty() {
+ let encode_accesses = encode_accesses.iter().fold(quote! {}, |a, b| {
+ if !a.is_empty() && !b.is_empty() {
+ quote! {#a , #b}
+ } else {
+ quote! {#a #b}
+ }
+ });
+ quote! { encoder.access(&[#encode_accesses].into_iter().flatten().collect::<Vec<_>>()); }
+ } else {
+ quote! {}
+ };
+
let (dispatch, elements) = if kernel.has_axis() {
if kernel.has_groups() || kernel.has_threads() {
anyhow::bail!("mixing groups/threads and axis is not supported");
@@ -245,10 +310,9 @@ pub fn bindgen(
(
quote! {
- let (__dsl_buffer, __dsl_offset) = __dsl_indirect_dispatch_buffer.into_parts();
compute_encoder.dispatch_threadgroups_indirect(
- __dsl_buffer,
- __dsl_offset,
+ __dsl_indirect_dispatch_buffer.0,
+ __dsl_indirect_dispatch_buffer.1,
MTLSize::new(#((#threads) as usize, )*),
);
},
@@ -337,9 +401,11 @@ pub fn bindgen(
Ok(Self { pipeline #(, #conditional_buffer_sets)* })
}
- #method_visibility fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {
- let compute_encoder = command_buffer.ensure_compute();
+ #method_visibility fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::common::Encoder<crate::backends::metal::Metal>) {
#empty_dispatch_guards
+ #(#encode_deconstructs)*
+ #encode_accesses
+ let compute_encoder = encoder.as_command_buffer_mut().ensure_compute();
compute_encoder.set_compute_pipeline_state(&self.pipeline);
#(#encode_args_sets)*
#dispatch
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
index cfafd4d6a..ae3db74d6 100644
--- a/crates/uzu/build/metal/compiler.rs
+++ b/crates/uzu/build/metal/compiler.rs
@@ -280,9 +280,9 @@ impl MetalCompiler {
use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};
use objc2::{rc::Retained, runtime::ProtocolObject};
+ use crate::backends::common::BufferGpuAddressRangeExt;
use crate::backends::metal::{
context::MetalContext,
- command_buffer::MetalCommandBufferEncoding,
error::MetalError,
metal_extensions::{
ComputeEncoderSetValue, FunctionConstantValuesSetValue, MetalDataTypeExt,
diff --git a/crates/uzu/src/backends/common/allocator/allocator.rs b/crates/uzu/src/backends/common/allocator/allocator.rs
new file mode 100644
index 000000000..093f37a37
--- /dev/null
+++ b/crates/uzu/src/backends/common/allocator/allocator.rs
@@ -0,0 +1,197 @@
+use std::{
+ cell::RefCell,
+ ops::Range,
+ pin::Pin,
+ rc::{Rc, Weak},
+};
+
+use super::{RangeAllocationType, RangeAllocator};
+use crate::backends::common::{Backend, Buffer, Context};
+
+pub struct Allocation<B: Backend> {
+ allocator: Rc<Allocator<B>>,
+ buffer: *const B::Buffer,
+ range: Range<usize>,
+}
+
+impl<B: Backend> Allocation<B> {
+ pub fn as_buffer_range<'a>(&'a self) -> (&'a B::Buffer, Range<usize>) {
+ (unsafe { &*self.buffer }, self.range.clone())
+ }
+}
+
+impl<B: Backend> Drop for Allocation<B> {
+ fn drop(&mut self) {
+ self.allocator.free(self)
+ }
+}
+
+pub struct AllocationPool<B: Backend> {
+ reusable: bool,
+ allocator: Rc<Allocator<B>>,
+ pool_number: usize,
+}
+
+impl<B: Backend> Drop for AllocationPool<B> {
+ fn drop(&mut self) {
+ self.allocator.free_pool(self)
+ }
+}
+
+pub enum AllocationType<'a, B: Backend> {
+ Global,
+ Pooled {
+ pool: &'a AllocationPool<B>,
+ cpu_available: bool,
+ },
+}
+
+struct AllocatorBuffer<B: Backend> {
+ buffer: Pin<Box<B::Buffer>>,
+ range_allocator: RangeAllocator<usize>,
+}
+
+pub struct Allocator<B: Backend> {
+ context: Weak<B::Context>,
+ allocator_buffers: RefCell<Vec<AllocatorBuffer<B>>>,
+ next_pool_number: RefCell<usize>,
+ peak_memory_usage: RefCell<usize>,
+}
+
+impl<B: Backend> Allocator<B> {
+ pub fn new(context: Weak<B::Context>) -> Rc<Self> {
+ Rc::new(Self {
+ context,
+ allocator_buffers: RefCell::new(Vec::new()),
+ next_pool_number: RefCell::new(0),
+ peak_memory_usage: RefCell::new(0),
+ })
+ }
+
+ pub fn allocate(
+ self: &Rc<Self>,
+ size: usize,
+ allocation_type: AllocationType<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let alignment = usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, 16_384);
+ let allocation_type = match allocation_type {
+ AllocationType::Global => RangeAllocationType::Global,
+ AllocationType::Pooled {
+ pool,
+ cpu_available,
+ } => RangeAllocationType::Pooled {
+ pool: pool.pool_number,
+ can_alias_before: !cpu_available,
+ can_alias_after: !(cpu_available && pool.reusable),
+ },
+ };
+
+ let mut allocator_buffers = self.allocator_buffers.borrow_mut();
+
+ let mut found: Option<(*const B::Buffer, Range<usize>)> = None;
+
+ for allocator_buffer in allocator_buffers.iter_mut() {
+ if let Some(range) =
+ allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type.clone())
+ {
+ found = Some((allocator_buffer.buffer.as_ref().get_ref() as *const B::Buffer, range));
+ break;
+ }
+ }
+
+ let (buffer, range) = if let Some((buffer, range)) = found {
+ (buffer, range)
+ } else {
+ let new_allocator_buffer_size = usize::max(size, 268_435_456);
+
+ let mut allocator_buffer = AllocatorBuffer::<B> {
+ buffer: Box::pin(self.context.upgrade().unwrap().create_buffer(new_allocator_buffer_size)?), // Upgrade can never fail
+ range_allocator: RangeAllocator::new(0..new_allocator_buffer_size),
+ };
+
+ let buffer = allocator_buffer.buffer.as_ref().get_ref() as *const B::Buffer;
+ let range =
+ allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type).unwrap(); // Can never fail
+
+ allocator_buffers.push(allocator_buffer);
+
+ *self.peak_memory_usage.borrow_mut() =
+ allocator_buffers.iter().map(|allocator_buffer| allocator_buffer.buffer.length()).sum();
+
+ (buffer, range)
+ };
+
+ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());
+
+ Ok(Allocation {
+ allocator: self.clone(),
+ buffer,
+ range,
+ })
+ }
+
+ pub fn create_pool(
+ self: &Rc<Self>,
+ reusable: bool,
+ ) -> AllocationPool<B> {
+ let pool_number = *self.next_pool_number.borrow();
+
+ *self.next_pool_number.borrow_mut() += 1;
+
+ AllocationPool {
+ reusable,
+ allocator: self.clone(),
+ pool_number,
+ }
+ }
+
+ pub fn peak_memory_usage(self: Rc<Self>) -> usize {
+ *self.peak_memory_usage.borrow()
+ }
+
+ // TODO: Maybe hysteresis in free/free_pool?
+
+ fn free(
+ self: &Rc<Self>,
+ allocation: &Allocation<B>,
+ ) {
+ let mut allocator_buffers = self.allocator_buffers.borrow_mut();
+
+ let (allocator_buffer_index, allocator_buffer) = allocator_buffers
+ .iter_mut()
+ .enumerate()
+ .find(|(_allocator_buffer_index, allocator_buffer)| {
+ (allocator_buffer.buffer.as_ref().get_ref() as *const B::Buffer) == allocation.buffer
+ })
+ .unwrap(); // Can never fail
+
+ allocator_buffer.range_allocator.free_range(allocation.range.clone());
+
+ if allocator_buffer.range_allocator.is_empty() {
+ allocator_buffers.remove(allocator_buffer_index);
+ }
+
+ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());
+ }
+
+ fn free_pool(
+ self: &Rc<Self>,
+ pool: &AllocationPool<B>,
+ ) {
+ let mut allocator_buffers = self.allocator_buffers.borrow_mut();
+ let mut free_idxs = Vec::new();
+
+ for (allocation_buffer_idx, allocation_buffer) in allocator_buffers.iter_mut().enumerate() {
+ allocation_buffer.range_allocator.free_pool(pool.pool_number);
+ if allocation_buffer.range_allocator.is_empty() {
+ free_idxs.push(allocation_buffer_idx);
+ }
+ }
+
+ for (i, free_idx) in free_idxs.into_iter().enumerate() {
+ allocator_buffers.remove(free_idx - i);
+ }
+
+ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());
+ }
+}
diff --git a/crates/uzu/src/backends/common/allocator/mod.rs b/crates/uzu/src/backends/common/allocator/mod.rs
new file mode 100644
index 000000000..c6e4a183c
--- /dev/null
+++ b/crates/uzu/src/backends/common/allocator/mod.rs
@@ -0,0 +1,5 @@
+mod allocator;
+mod range_allocator;
+
+pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};
+use range_allocator::{AllocationType as RangeAllocationType, RangeAllocator};
diff --git a/crates/uzu/src/backends/common/allocator/range_allocator.rs b/crates/uzu/src/backends/common/allocator/range_allocator.rs
new file mode 100644
index 000000000..635729787
--- /dev/null
+++ b/crates/uzu/src/backends/common/allocator/range_allocator.rs
@@ -0,0 +1,197 @@
+use std::ops::Range;
+
+use rangemap::{RangeMap, RangeSet};
+
+#[derive(PartialEq, Eq, Clone)]
+pub enum AllocationType<P> {
+ Global,
+ Pooled {
+ pool: P,
+ can_alias_before: bool,
+ can_alias_after: bool,
+ },
+}
+
+#[derive(PartialEq, Eq, Clone)]
+enum PooledAllocationRangeStatus {
+ Allocated {
+ can_alias_after: bool,
+ },
+ Aliasable,
+ Frozen,
+}
+
+#[derive(PartialEq, Eq, Clone)]
+enum AllocationRangeType<P: Eq + Clone> {
+ Global,
+ Pooled {
+ pool: P,
+ status: PooledAllocationRangeStatus,
+ },
+}
+
+pub struct RangeAllocator<P: Eq + Clone> {
+ full_range: Range<usize>,
+ ranges: RangeMap<usize, AllocationRangeType<P>>,
+}
+
+impl<P: Eq + Clone> RangeAllocator<P> {
+ pub fn new(full_range: Range<usize>) -> Self {
+ Self {
+ full_range,
+ ranges: RangeMap::new(),
+ }
+ }
+
+ pub fn allocate_range_aligned(
+ &mut self,
+ size: usize,
+ alignment: usize,
+ allocation_type: AllocationType<P>,
+ ) -> Option<Range<usize>> {
+ let mut available_range = RangeSet::new();
+ available_range.extend(self.ranges.gaps(&self.full_range));
+
+ if let AllocationType::Pooled {
+ pool,
+ can_alias_before: true,
+ can_alias_after: _,
+ } = &allocation_type
+ {
+ available_range.extend(self.ranges.iter().flat_map(|(range, range_type)| {
+ if let AllocationRangeType::Pooled {
+ pool: range_pool,
+ status: PooledAllocationRangeStatus::Aliasable,
+ } = range_type
+ && range_pool == pool
+ {
+ Some(range.clone())
+ } else {
+ None
+ }
+ }));
+ }
+
+ let Some(smallest_fitting_free_range) = available_range
+ .iter()
+ .filter(|range| range.end.saturating_sub(range.start.next_multiple_of(alignment)) >= size)
+ .min_by_key(|range| range.len())
+ else {
+ return None;
+ };
+
+ let allocated_range_start = smallest_fitting_free_range.start.next_multiple_of(alignment);
+ let allocated_range = allocated_range_start..(allocated_range_start + size);
+ let allocated_range_status = match allocation_type {
+ AllocationType::Global => AllocationRangeType::Global,
+ AllocationType::Pooled {
+ pool,
+ can_alias_before: _,
+ can_alias_after,
+ } => AllocationRangeType::Pooled {
+ pool,
+ status: PooledAllocationRangeStatus::Allocated {
+ can_alias_after,
+ },
+ },
+ };
+
+ self.ranges.insert(allocated_range.clone(), allocated_range_status);
+
+ Some(allocated_range)
+ }
+
+ pub fn free_range(
+ &mut self,
+ range: Range<usize>,
+ ) {
+ let overlapping_ranges = self.ranges.overlapping(range.clone()).collect::<Vec<_>>();
+ assert!(overlapping_ranges.len() == 1, "expected only 1 overlapping range");
+
+ let (overlapping_range, range_type) = overlapping_ranges[0];
+ assert!(
+ overlapping_range.start <= range.start && overlapping_range.end >= range.end,
+ "overlapping range doesn't fully cover the range"
+ );
+
+ match range_type {
+ AllocationRangeType::Global => self.ranges.remove(range),
+ AllocationRangeType::Pooled {
+ pool,
+ status,
+ } => {
+ let status = match status {
+ PooledAllocationRangeStatus::Allocated {
+ can_alias_after: true,
+ } => PooledAllocationRangeStatus::Aliasable,
+ PooledAllocationRangeStatus::Allocated {
+ can_alias_after: false,
+ } => PooledAllocationRangeStatus::Frozen,
+ _ => panic!("attempted to free a range that isn't allocated"),
+ };
+
+ self.ranges.insert(
+ range,
+ AllocationRangeType::Pooled {
+ pool: pool.clone(),
+ status,
+ },
+ )
+ },
+ };
+ }
+
+ pub fn free_pool(
+ &mut self,
+ pool: P,
+ ) {
+ let pool_ranges = self
+ .ranges
+ .iter()
+ .filter_map(|(range, range_type)| {
+ if let AllocationRangeType::Pooled {
+ pool: range_pool,
+ status,
+ } = range_type
+ && range_pool == &pool
+ {
+ assert!(
+ !matches!(status, PooledAllocationRangeStatus::Allocated { .. }),
+ "attempted to free a pool that has live allocations"
+ );
+ Some(range.clone())
+ } else {
+ None
+ }
+ })
+ .collect::<Vec<_>>();
+
+ for pool_range in pool_ranges {
+ self.ranges.remove(pool_range);
+ }
+ }
+
+ pub fn total_available(&self) -> usize {
+ self.ranges
+ .gaps(&self.full_range)
+ .map(|gap| gap.len())
+ .chain(self.ranges.iter().filter_map(|(range, range_type)| {
+ if matches!(
+ range_type,
+ AllocationRangeType::Pooled {
+ pool: _,
+ status: PooledAllocationRangeStatus::Aliasable
+ }
+ ) {
+ Some(range.len())
+ } else {
+ None
+ }
+ }))
+ .sum()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.ranges.is_empty()
+ }
+}
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rs
index e7bba72f3..e0f18747a 100644
--- a/crates/uzu/src/backends/common/backend.rs
+++ b/crates/uzu/src/backends/common/backend.rs
@@ -10,5 +10,6 @@ pub trait Backend: Debug + Clone + 'static {
type Kernels: Kernels<Backend = Self> + MatmulKernels;
type Error: Error + Debug;
+ const MIN_ALLOCATION_ALIGNMENT: usize;
const MAX_INLINE_BYTES: usize;
}
diff --git a/crates/uzu/src/backends/common/buffer.rs b/crates/uzu/src/backends/common/buffer.rs
index 832ce2cc9..2a628617f 100644
--- a/crates/uzu/src/backends/common/buffer.rs
+++ b/crates/uzu/src/backends/common/buffer.rs
@@ -1,4 +1,4 @@
-use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};
+use std::{fmt::Debug, ops::Range, os::raw::c_void, ptr::NonNull};
use super::Backend;
@@ -11,8 +11,24 @@ pub trait Buffer: Debug {
);
fn cpu_ptr(&self) -> NonNull<c_void>;
+ fn gpu_ptr(&self) -> usize;
fn length(&self) -> usize;
+}
+
+pub trait BufferGpuAddressRangeExt: Buffer {
+ fn gpu_address_range(&self) -> Range<usize> {
+ self.gpu_ptr()..(self.gpu_ptr() + self.length())
+ }
- fn id(&self) -> usize;
+ fn gpu_address_subrange(
+ &self,
+ subrange: Range<usize>,
+ ) -> Range<usize> {
+ assert!(subrange.end <= self.length(), "subrange overflow: subrange={:?} length={}", subrange, self.length());
+
+ (self.gpu_ptr() + subrange.start)..(self.gpu_ptr() + subrange.end)
+ }
}
+
+impl<B: Buffer> BufferGpuAddressRangeExt for B {}
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index d597c899b..523aff8cf 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -18,6 +18,55 @@ pub trait CommandBufferInitial {
fn start_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Encoding;
}
+#[derive(Debug, Clone, PartialEq)]
+pub struct AccessFlags {
+ pub compute_read: bool,
+ pub compute_write: bool,
+ pub copy_read: bool,
+ pub copy_write: bool,
+}
+
+impl AccessFlags {
+ pub fn empty() -> Self {
+ Self {
+ compute_read: false,
+ compute_write: false,
+ copy_read: false,
+ copy_write: false,
+ }
+ }
+
+ pub fn with_compute_read(mut self) -> Self {
+ self.compute_read = true;
+ self
+ }
+ pub fn with_compute_write(mut self) -> Self {
+ self.compute_write = true;
+ self
+ }
+ pub fn with_copy_read(mut self) -> Self {
+ self.copy_read = true;
+ self
+ }
+ pub fn with_copy_write(mut self) -> Self {
+ self.copy_write = true;
+ self
+ }
+
+ pub fn compute_read() -> Self {
+ Self::empty().with_compute_read()
+ }
+ pub fn compute_write() -> Self {
+ Self::empty().with_compute_write()
+ }
+ pub fn copy_read() -> Self {
+ Self::empty().with_copy_read()
+ }
+ pub fn copy_write() -> Self {
+ Self::empty().with_copy_write()
+ }
+}
+
pub trait CommandBufferEncoding {
type CommandBuffer: CommandBuffer<Encoding = Self>;
@@ -35,6 +84,12 @@ pub trait CommandBufferEncoding {
value: u8,
);
+ fn encode_barrier(
+ &mut self,
+ after: AccessFlags,
+ before: AccessFlags,
+ );
+
fn encode_wait_for_event(
&mut self,
event: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Event,
diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rs
index 6daca784b..ddcd74db1 100644
--- a/crates/uzu/src/backends/common/context.rs
+++ b/crates/uzu/src/backends/common/context.rs
@@ -1,7 +1,6 @@
use std::{env, path::Path, rc::Rc};
-use super::Backend;
-use crate::backends::common::CommandBuffer;
+use crate::backends::common::{Allocation, AllocationPool, AllocationType, Backend, CommandBuffer};
pub trait Context: Sized {
type Backend: Backend<Context = Self>;
@@ -26,6 +25,17 @@ pub trait Context: Sized {
size: usize,
) -> Result<<Self::Backend as Backend>::Buffer, <Self::Backend as Backend>::Error>;
+ fn create_allocation(
+ &self,
+ size: usize,
+ allocation_type: AllocationType<Self::Backend>,
+ ) -> Result<Allocation<Self::Backend>, <Self::Backend as Backend>::Error>;
+
+ fn create_allocation_pool(
+ &self,
+ reusable: bool,
+ ) -> AllocationPool<Self::Backend>;
+
fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;
fn enable_capture();
diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rs
new file mode 100644
index 000000000..7c0136060
--- /dev/null
+++ b/crates/uzu/src/backends/common/encoder.rs
@@ -0,0 +1,173 @@
+use std::{ops::Range, time::Duration};
+
+use crate::backends::common::{
+ AccessFlags, Allocation, AllocationPool, AllocationType, Backend, BufferGpuAddressRangeExt, CommandBuffer,
+ CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
+ Context,
+ hazard_tracker::{Access, HazardTracker},
+};
+
+pub struct Encoder<'encoding, B: Backend> {
+ context: &'encoding B::Context,
+ command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,
+ allocation_pool: AllocationPool<B>,
+ hazard_tracker: HazardTracker,
+}
+
+impl<'encoding, B: Backend> Encoder<'encoding, B> {
+ pub fn new(context: &'encoding B::Context) -> Result<Self, B::Error> {
+ let command_buffer = context.create_command_buffer()?.start_encoding();
+ let allocation_pool = context.create_allocation_pool(false);
+ let hazard_tracker = HazardTracker::new();
+
+ Ok(Self {
+ context,
+ command_buffer,
+ allocation_pool,
+ hazard_tracker,
+ })
+ }
+
+ // This is valid on both cpu and gpu timelines
+ pub fn allocate_constant(
+ &mut self,
+ size: usize,
+ ) -> Result<Allocation<B>, B::Error> {
+ self.context.create_allocation(
+ size,
+ AllocationType::Pooled {
+ pool: &self.allocation_pool,
+ cpu_available: true,
+ },
+ )
+ }
+
+ // This is valid on gpu timeline only
+ pub fn allocate_scratch(
+ &mut self,
+ size: usize,
+ ) -> Result<Allocation<B>, B::Error> {
+ self.context.create_allocation(
+ size,
+ AllocationType::Pooled {
+ pool: &self.allocation_pool,
+ cpu_available: false,
+ },
+ )
+ }
+
+ pub fn encode_copy(
+ &mut self,
+ src: &B::Buffer,
+ dst: &mut B::Buffer,
+ size: usize,
+ ) {
+ self.access(&[
+ Access {
+ range: src.gpu_address_subrange(0..size),
+ flags: AccessFlags::copy_read(),
+ },
+ Access {
+ range: dst.gpu_address_subrange(0..size),
+ flags: AccessFlags::copy_write(),
+ },
+ ]);
+ self.command_buffer.encode_copy(src, dst, size);
+ }
+
+ pub fn encode_fill(
+ &mut self,
+ dst: &mut B::Buffer,
+ range: Range<usize>,
+ value: u8,
+ ) {
+ self.access(&[Access {
+ range: dst.gpu_address_subrange(range.clone()),
+ flags: AccessFlags::copy_write(),
+ }]);
+ self.command_buffer.encode_fill(dst, range, value);
+ }
+
+ pub fn access(
+ &mut self,
+ accesses: &[Access],
+ ) {
+ if let Some((after, before)) = self.hazard_tracker.access(accesses) {
+ self.command_buffer.encode_barrier(after, before);
+ }
+ }
+
+ pub fn encode_wait_for_event(
+ &mut self,
+ event: &B::Event,
+ value: u64,
+ ) {
+ self.command_buffer.encode_wait_for_event(event, value);
+ }
+
+ pub fn encode_signal_event(
+ &mut self,
+ event: &B::Event,
+ value: u64,
+ ) {
+ self.command_buffer.encode_signal_event(event, value);
+ }
+
+ pub fn add_completion_handler(
+ &mut self,
+ handler: impl FnOnce(Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>) + 'static,
+ ) {
+ self.command_buffer.add_completion_handler(handler);
+ }
+
+ pub fn as_command_buffer_mut(&mut self) -> &mut <B::CommandBuffer as CommandBuffer>::Encoding {
+ &mut self.command_buffer
+ }
+
+ pub fn end_encoding(self) -> Executable<B> {
+ Executable {
+ command_buffer: self.command_buffer.end_encoding(),
+ allocation_pool: self.allocation_pool,
+ }
+ }
+}
+
+pub struct Executable<B: Backend> {
+ command_buffer: <B::CommandBuffer as CommandBuffer>::Executable,
+ allocation_pool: AllocationPool<B>,
+}
+
+impl<B: Backend> Executable<B> {
+ pub fn submit(self) -> Pending<B> {
+ Pending {
+ command_buffer: self.command_buffer.submit(),
+ allocation_pool: self.allocation_pool,
+ }
+ }
+}
+
+pub struct Pending<B: Backend> {
+ command_buffer: <B::CommandBuffer as CommandBuffer>::Pending,
+ allocation_pool: AllocationPool<B>,
+}
+
+impl<B: Backend> Pending<B> {
+ pub fn wait_until_completed(self) -> Result<Completed<B>, B::Error> {
+ Ok(Completed {
+ command_buffer: self.command_buffer.wait_until_completed()?,
+ allocation_pool: self.allocation_pool,
+ })
+ }
+}
+
+pub struct Completed<B: Backend> {
+ command_buffer: <B::CommandBuffer as CommandBuffer>::Completed,
+ #[allow(unused)]
+ allocation_pool: AllocationPool<B>,
+}
+
+impl<B: Backend> Completed<B> {
+ pub fn gpu_execution_time(&self) -> Option<Duration> {
+ self.command_buffer.gpu_execution_time()
+ }
+}
diff --git a/crates/uzu/src/backends/common/hazard_tracker.rs b/crates/uzu/src/backends/common/hazard_tracker.rs
new file mode 100644
index 000000000..fc1ea6f4c
--- /dev/null
+++ b/crates/uzu/src/backends/common/hazard_tracker.rs
@@ -0,0 +1,98 @@
+use std::ops::Range;
+
+use rangemap::RangeSet;
+
+use crate::backends::common::AccessFlags;
+
+#[derive(Debug, Clone)]
+pub struct Access {
+ pub range: Range<usize>,
+ pub flags: AccessFlags,
+}
+
+pub struct HazardTracker {
+ compute_reads: RangeSet<usize>,
+ compute_writes: RangeSet<usize>,
+ copy_reads: RangeSet<usize>,
+ copy_writes: RangeSet<usize>,
+}
+
+impl HazardTracker {
+ pub fn new() -> Self {
+ Self {
+ compute_reads: RangeSet::new(),
+ compute_writes: RangeSet::new(),
+ copy_reads: RangeSet::new(),
+ copy_writes: RangeSet::new(),
+ }
+ }
+
+ #[must_use]
+ pub fn access(
+ &mut self,
+ accesses: &[Access],
+ ) -> Option<(AccessFlags, AccessFlags)> {
+ let mut after = AccessFlags::empty();
+ let mut before = AccessFlags::empty();
+
+ // Get existing hazards (before the barrier, aka barrier after this)
+ for access in accesses {
+ // Any access should wait for earlier overlapping writes to complete (RAW/WAW)
+ if access.flags.compute_read
+ || access.flags.compute_write
+ || access.flags.copy_read
+ || access.flags.copy_write
+ {
+ if self.compute_writes.overlaps(&access.range) {
+ after.compute_write = true;
+ self.compute_writes.clear();
+ }
+ if self.copy_writes.overlaps(&access.range) {
+ after.copy_write = true;
+ self.copy_writes.clear();
+ }
+ }
+ // Write accesses also must wait for previous reads to complete (WAR)
+ if access.flags.compute_write || access.flags.copy_write {
+ if self.compute_reads.overlaps(&access.range) {
+ after.compute_read = true;
+ self.compute_reads.clear();
+ }
+ if self.copy_reads.overlaps(&access.range) {
+ after.copy_read = true;
+ self.copy_reads.clear();
+ }
+ }
+ }
+
+ // Track new ones (after the barrier, aka barrier before this)
+ for access in accesses {
+ if access.flags.compute_read {
+ before.compute_read = true;
+ self.compute_reads.insert(access.range.clone());
+ }
+ if access.flags.compute_write {
+ before.compute_write = true;
+ self.compute_writes.insert(access.range.clone());
+ }
+ if access.flags.copy_read {
+ before.copy_read = true;
+ self.copy_reads.insert(access.range.clone());
+ }
+ if access.flags.copy_write {
+ before.copy_write = true;
+ self.copy_writes.insert(access.range.clone());
+ }
+ }
+
+ if after.compute_read || after.compute_write || after.copy_read || after.copy_write {
+ Some((after, before))
+ } else {
+ None
+ }
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../tests_unit/hazard_tracker_test.rs"]
+mod hazard_tracker_test;
diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rs
index c0522bdd7..339d104fd 100644
--- a/crates/uzu/src/backends/common/kernel/attention.rs
+++ b/crates/uzu/src/backends/common/kernel/attention.rs
@@ -6,7 +6,7 @@ use std::{
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
gpu_types::{AttnMaskParams, AttnParams},
kernel::AttentionGemmKernel,
},
@@ -49,7 +49,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
pub fn encode(
&self,
context: &B::Context,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: AttentionGemmArguments<B>,
) -> Result<(), B::Error> {
let bk: usize = if args.head_dim < 128 {
@@ -138,7 +138,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
args.sinks_buffer,
args.num_heads as u32,
args.suffix_length as u32,
- command_buffer,
+ encoder,
);
Ok(())
diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rs
index 04692fa70..f7be333b6 100644
--- a/crates/uzu/src/backends/common/kernel/audio.rs
+++ b/crates/uzu/src/backends/common/kernel/audio.rs
@@ -1,4 +1,4 @@
-use crate::backends::common::{Backend, CommandBuffer};
+use crate::backends::common::{Backend, Encoder};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AudioPadMode {
@@ -120,68 +120,68 @@ pub struct AudioClampArguments<'a, B: Backend> {
pub trait AudioKernelRuntime<B: Backend> {
fn encode_fsq_decode(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioFsqDecodeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_fsq_encode(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioFsqEncodeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_conv1d(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioConv1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_causal_conv1d(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioCausalConv1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_causal_conv_transpose1d(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioCausalConvTranspose1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_half_snake(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioHalfSnakeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_leaky_relu(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioElementwiseArguments<'_, B>,
negative_slope: f32,
) -> Result<(), B::Error>;
fn encode_tanh(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioElementwiseArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_add(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioAddArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_scale(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioScaleArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_clamp(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: AudioClampArguments<'_, B>,
) -> Result<(), B::Error>;
}
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
index d208d7156..daedd5865 100644
--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -4,7 +4,7 @@ use thiserror::Error;
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
+ backends::common::{Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
};
pub struct KVLayerData<B: Backend> {
@@ -58,7 +58,7 @@ impl<B: Backend> KVCacheUpdate<B> {
in_place_data: &[KVLayerData<B>],
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), KVCacheUpdateError<B>> {
if source_indices.len() != destination_indices.len() {
return Err(KVCacheUpdateError::IndicesCountMismatch);
@@ -87,7 +87,7 @@ impl<B: Backend> KVCacheUpdate<B> {
num_heads as u32,
max_sequence_length as u32,
head_dim as u32,
- command_buffer,
+ encoder,
);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
index f1c793475..78a99f7d8 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
@@ -6,7 +6,7 @@ use thiserror::Error;
use super::Kernels;
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, Encoder},
};
pub trait MatmulKernels: Kernels {
@@ -25,7 +25,7 @@ pub trait MatmulKernel: Sized {
&mut self,
context: &<Self::Backend as Backend>::Context,
arguments: MatmulArguments<Self::Backend>,
- command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<Self::Backend>,
);
}
diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
index 2c5f1863a..0561ac39f 100644
--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -1,7 +1,7 @@
use crate::{
DataType,
backends::common::{
- ActivationConfig, Backend, CommandBuffer, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
+ ActivationConfig, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
},
};
@@ -28,7 +28,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
pub fn encode(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
fused_up: &B::Buffer,
hidden: &mut B::Buffer,
m: i32,
@@ -36,7 +36,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
if self.activation.act_type() == ActivationType::IDENTITY {
panic!("Identity activation is not supported for kernel")
}
- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), command_buffer);
+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), encoder);
Ok(())
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
index bcaa38a61..484cb916b 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
@@ -3,7 +3,7 @@ use std::ops::{Deref, DerefMut};
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
kernel::{MoeExpertsDecodeSinglePassAKernel, MoeExpertsDecodeSinglePassBKernel},
},
};
@@ -84,7 +84,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
pub fn encode(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
mut args: MoeExpertsSingleDecodeArguments<B>,
) {
if args.k == 0 {
@@ -110,7 +110,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
args.gate_clip_max,
args.up_clip_min,
args.up_clip_max,
- command_buffer,
+ encoder,
);
// Pass B: 8 simdgroups (256 threads), outputs final y directly
@@ -125,7 +125,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
args.d_model as u32,
args.d_ff as u32,
args.k as u32,
- command_buffer,
+ encoder,
);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
index 1030d6a5c..0749d385e 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
@@ -7,7 +7,7 @@ use super::{
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},
},
};
@@ -47,7 +47,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
pub fn encode(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
mut args: MoeExpertsTwoPassArguments<B>,
) {
if args.total_rows == 0 {
@@ -58,7 +58,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
const BLOCK_M: u32 = 4;
let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;
self.pass_a_tile.encode_counts(
- command_buffer,
+ encoder,
MoePassATileCountsArguments {
expert_offsets: args.expert_offsets,
tile_counts: args.tile_counts.deref_mut(),
@@ -67,7 +67,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
},
);
self.pass_a_tile.encode_scan(
- command_buffer,
+ encoder,
MoePassATileScanArguments {
tile_counts: args.tile_counts,
tile_offsets: args.tile_offsets.deref_mut(),
@@ -76,7 +76,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
},
);
self.pass_a_tile.encode_row_map(
- command_buffer,
+ encoder,
MoePassARowMapArguments {
expert_offsets: args.expert_offsets,
row_expert_map: args.row_expert_map.deref_mut(),
@@ -85,7 +85,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
},
);
self.pass_a_tile.encode_build_map(
- command_buffer,
+ encoder,
MoePassATileBuildArguments {
expert_offsets: args.expert_offsets,
tile_offsets: args.tile_offsets,
@@ -96,7 +96,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
},
);
self.pass_a_tile.encode_dispatch_args(
- command_buffer,
+ encoder,
MoePassATileDispatchArguments {
total_tiles: args.total_tiles,
dispatch_args: args.dispatch_args.deref_mut(),
@@ -125,7 +125,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
args.silu_alpha,
args.tile_map.deref(),
args.dispatch_args.deref(),
- command_buffer,
+ encoder,
);
// pass b
@@ -140,7 +140,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
args.d_model as u32,
args.d_ff as u32,
args.e as u32,
- command_buffer,
+ encoder,
);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
index 15cd2e280..4cae93202 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
@@ -6,7 +6,7 @@ use super::{
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, CommandBufferEncoding, Kernels,
+ Backend, Encoder, Kernels,
kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},
},
};
@@ -48,7 +48,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
pub fn encode(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
mut args: MoeExpertsTwoPassArguments<B>,
) {
if args.total_rows == 0 {
@@ -56,10 +56,10 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
}
let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
- command_buffer.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);
+ encoder.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);
self.tile_map.encode_counts(
- command_buffer,
+ encoder,
MoeTileCountsArguments {
offsets_buffer: args.expert_offsets,
tile_counts_buffer: args.tile_counts.deref_mut(),
@@ -67,7 +67,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
},
);
self.tile_map.encode_scan(
- command_buffer,
+ encoder,
MoeTileScanArguments {
tile_counts_buffer: args.tile_counts,
tile_offsets_buffer: args.tile_offsets.deref_mut(),
@@ -76,7 +76,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
},
);
self.tile_map.encode_build_map(
- command_buffer,
+ encoder,
MoeTileMapBuildArguments {
expert_offsets: args.expert_offsets,
tile_offsets: args.tile_offsets,
@@ -100,7 +100,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
(d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL
};
self.tile_map.encode_dispatch_args(
- command_buffer,
+ encoder,
MoeTileDispatchArguments {
total_tiles: args.total_tiles,
dispatch_args: args.dispatch_args.deref_mut(),
@@ -128,7 +128,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args.silu_alpha,
args.tile_map.deref(),
args.dispatch_args.deref(),
- command_buffer,
+ encoder,
);
let dispatch_args = MoeTileDispatchArguments {
@@ -136,7 +136,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
dispatch_args: args.dispatch_args.deref_mut(),
num_tiles_x: n_tiles_model,
};
- self.tile_map.encode_dispatch_args(command_buffer, dispatch_args);
+ self.tile_map.encode_dispatch_args(encoder, dispatch_args);
let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
kernel_pass_b.encode(
@@ -150,7 +150,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args.e as u32,
args.tile_map.deref(),
args.dispatch_args.deref(),
- command_buffer,
+ encoder,
);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rs
index 771c32ada..fc7c2cf8f 100644
--- a/crates/uzu/src/backends/common/kernel/moe/gather.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs
@@ -1,7 +1,7 @@
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
},
};
@@ -34,7 +34,7 @@ impl<B: Backend> MoeGatherKernels<B> {
pub fn encode(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
dtype: DataType,
args: MoeGatherArguments<B>,
) {
@@ -47,7 +47,7 @@ impl<B: Backend> MoeGatherKernels<B> {
args.d_model as u32,
args.t as u32,
args.k as u32,
- command_buffer,
+ encoder,
),
DataType::F16 => self.f16.encode(
args.x_buffer,
@@ -57,7 +57,7 @@ impl<B: Backend> MoeGatherKernels<B> {
args.d_model as u32,
args.t as u32,
args.k as u32,
- command_buffer,
+ encoder,
),
DataType::BF16 => self.bf16.encode(
args.x_buffer,
@@ -67,7 +67,7 @@ impl<B: Backend> MoeGatherKernels<B> {
args.d_model as u32,
args.t as u32,
args.k as u32,
- command_buffer,
+ encoder,
),
_ => panic!("Unsupported data type: {:?}", dtype),
};
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
index ea144b317..e1c068c24 100644
--- a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
@@ -1,5 +1,5 @@
use crate::backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},
};
@@ -22,15 +22,15 @@ impl<B: Backend> MoeTileMapKernels<B> {
pub fn encode_counts(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoeTileCountsArguments<B>,
) {
- self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, command_buffer);
+ self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);
}
pub fn encode_scan(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoeTileScanArguments<B>,
) {
self.scan.encode(
@@ -38,13 +38,13 @@ impl<B: Backend> MoeTileMapKernels<B> {
args.tile_offsets_buffer,
args.total_tiles_buffer,
args.e as u32,
- command_buffer,
+ encoder,
);
}
pub fn encode_build_map(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoeTileMapBuildArguments<B>,
) {
self.build.encode(
@@ -53,16 +53,16 @@ impl<B: Backend> MoeTileMapKernels<B> {
args.tile_counts,
args.tile_map,
args.e as u32,
- command_buffer,
+ encoder,
);
}
pub fn encode_dispatch_args(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoeTileDispatchArguments<B>,
) {
- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, command_buffer);
+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
index 22353c53c..69c70c669 100644
--- a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
@@ -1,5 +1,5 @@
use crate::backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{
Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,
MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,
@@ -68,37 +68,31 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_counts(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoePassATileCountsArguments<B>,
) {
- self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, command_buffer);
+ self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);
}
pub fn encode_scan(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoePassATileScanArguments<B>,
) {
- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, command_buffer);
+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);
}
pub fn encode_row_map(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoePassARowMapArguments<B>,
) {
- self.row_map.encode(
- args.expert_offsets,
- args.row_expert_map,
- args.total_rows as u32,
- args.e as u32,
- command_buffer,
- );
+ self.row_map.encode(args.expert_offsets, args.row_expert_map, args.total_rows as u32, args.e as u32, encoder);
}
pub fn encode_build_map(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoePassATileBuildArguments<B>,
) {
self.build_map.encode(
@@ -108,15 +102,15 @@ impl<B: Backend> MoePassATileKernels<B> {
args.tile_map,
args.total_rows as u32,
args.h_blocks,
- command_buffer,
+ encoder,
);
}
pub fn encode_dispatch_args(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: MoePassATileDispatchArguments<B>,
) {
- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, command_buffer);
+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);
}
}
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index ee92a9b76..4e3220a24 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -3,7 +3,7 @@ use std::collections::HashMap;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
gpu_types::QuantizationMode,
kernel::{
QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
@@ -161,7 +161,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
pub fn encode(
&self,
- encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
arguments: QuantizedMatmulArguments<B>,
) -> Result<(), QuantizedMatmulError<B>> {
if arguments.quantization_type != self.quantization_type {
diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rs
index 921b4b5a9..400faa501 100644
--- a/crates/uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/uzu/src/backends/common/kernel/sampling.rs
@@ -9,7 +9,7 @@ use thiserror::Error;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, Context, Kernels,
+ Backend, Context, Encoder, Kernels,
gpu_types::ArgmaxPair,
kernel::{
ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,
@@ -152,7 +152,7 @@ impl<B: Backend> SamplingKernel<B> {
sampling_method: SamplingMethod,
batch_size: usize,
vocab_size: usize,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), SamplingError<B>> {
if batch_size > self.max_batch_size {
return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));
@@ -168,7 +168,7 @@ impl<B: Backend> SamplingKernel<B> {
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
- command_buffer,
+ encoder,
);
}
@@ -186,7 +186,7 @@ impl<B: Backend> SamplingKernel<B> {
batch_size as u32,
vocab_size as u32,
temperature,
- command_buffer,
+ encoder,
);
}
@@ -197,7 +197,7 @@ impl<B: Backend> SamplingKernel<B> {
batch_size as u32,
vocab_size as u32,
top_k,
- command_buffer,
+ encoder,
);
}
@@ -208,7 +208,7 @@ impl<B: Backend> SamplingKernel<B> {
batch_size as u32,
vocab_size as u32,
top_p,
- command_buffer,
+ encoder,
);
}
@@ -219,7 +219,7 @@ impl<B: Backend> SamplingKernel<B> {
batch_size as u32,
vocab_size as u32,
min_p,
- command_buffer,
+ encoder,
);
}
@@ -229,7 +229,7 @@ impl<B: Backend> SamplingKernel<B> {
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
- command_buffer,
+ encoder,
);
}
@@ -242,7 +242,7 @@ impl<B: Backend> SamplingKernel<B> {
sampled_tokens_buffer,
batch_size as u32,
vocab_size as u32,
- command_buffer,
+ encoder,
);
},
ArgmaxImplementation::TwoPass {
@@ -255,14 +255,14 @@ impl<B: Backend> SamplingKernel<B> {
partial_results_buffer.borrow_mut().deref_mut(),
batch_size as u32,
vocab_size as u32,
- command_buffer,
+ encoder,
);
final_kernel.encode(
partial_results_buffer.borrow().deref(),
sampled_tokens_buffer,
batch_size as u32,
vocab_size as u32,
- command_buffer,
+ encoder,
);
},
}
diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
index cb57d9f33..7d944cf18 100644
--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -1,7 +1,7 @@
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},
},
};
@@ -55,7 +55,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
pub fn encode(
&self,
- compute_encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
args: SSDPrefillArguments<B>,
mode: SSDPrefillMode,
) {
@@ -84,7 +84,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
state_strides.as_slice(),
args.channels as u32,
args.head_dim as u32,
- compute_encoder,
+ encoder,
)
} else {
self.single.encode(
@@ -105,7 +105,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
state_strides.as_slice(),
args.channels as u32,
args.head_dim as u32,
- compute_encoder,
+ encoder,
)
}
} else if mode == SSDPrefillMode::Sequential {
@@ -127,7 +127,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
state_strides.as_slice(),
args.channels as u32,
args.head_dim as u32,
- compute_encoder,
+ encoder,
)
}
}
diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rs
index 8557fd41d..d31596649 100644
--- a/crates/uzu/src/backends/common/mod.rs
+++ b/crates/uzu/src/backends/common/mod.rs
@@ -1,19 +1,25 @@
mod activation_config;
+mod allocator;
mod backend;
mod buffer;
mod command_buffer;
mod context;
+mod encoder;
mod event;
pub mod gpu_types;
+mod hazard_tracker;
pub mod kernel;
pub use activation_config::ActivationConfig;
+pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};
pub use backend::Backend;
-pub use buffer::Buffer;
+pub use buffer::{Buffer, BufferGpuAddressRangeExt};
pub use command_buffer::{
- CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending,
+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
+ CommandBufferInitial, CommandBufferPending,
};
pub use context::Context;
+pub use encoder::{Completed, Encoder, Executable, Pending};
pub use event::Event;
+pub use hazard_tracker::Access;
pub use kernel::Kernels;
diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rs
index d6a2a5a6e..9d1e5e665 100644
--- a/crates/uzu/src/backends/cpu/backend.rs
+++ b/crates/uzu/src/backends/cpu/backend.rs
@@ -1,4 +1,4 @@
-use std::cell::RefCell;
+use std::cell::{RefCell, UnsafeCell};
use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};
use crate::backends::common::Backend;
@@ -9,10 +9,11 @@ pub struct Cpu;
impl Backend for Cpu {
type Context = CpuContext;
type CommandBuffer = CpuCommandBuffer;
- type Buffer = Box<[u8]>;
+ type Buffer = UnsafeCell<Box<[u8]>>;
type Event = RefCell<u64>;
type Kernels = CpuKernels;
type Error = CpuError;
+ const MIN_ALLOCATION_ALIGNMENT: usize = 64;
const MAX_INLINE_BYTES: usize = usize::MAX;
}
diff --git a/crates/uzu/src/backends/cpu/buffer.rs b/crates/uzu/src/backends/cpu/buffer.rs
index 306bc05e5..b147cf202 100644
--- a/crates/uzu/src/backends/cpu/buffer.rs
+++ b/crates/uzu/src/backends/cpu/buffer.rs
@@ -1,9 +1,9 @@
-use std::{os::raw::c_void, ptr::NonNull};
+use std::{cell::UnsafeCell, os::raw::c_void, ptr::NonNull};
use super::Cpu;
use crate::backends::common::Buffer;
-impl Buffer for Box<[u8]> {
+impl Buffer for UnsafeCell<Box<[u8]>> {
type Backend = Cpu;
fn set_label(
@@ -13,12 +13,14 @@ impl Buffer for Box<[u8]> {
}
fn cpu_ptr(&self) -> NonNull<c_void> {
- unsafe { NonNull::new_unchecked(self.as_ptr() as *mut c_void) }
+ unsafe { NonNull::new_unchecked((&*self.get()).as_ptr() as *mut c_void) }
}
- fn length(&self) -> usize {
- self.len()
+
+ fn gpu_ptr(&self) -> usize {
+ unsafe { &*self.get() }.as_ptr().addr()
}
- fn id(&self) -> usize {
- unimplemented!()
+
+ fn length(&self) -> usize {
+ unsafe { &*self.get() }.len()
}
}
diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rs
index 1896a56f1..8ce9063bd 100644
--- a/crates/uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu/src/backends/cpu/command_buffer.rs
@@ -1,13 +1,13 @@
use std::{
- cell::{OnceCell, RefCell},
+ cell::{OnceCell, RefCell, UnsafeCell},
time::{Duration, Instant},
};
use super::Cpu;
use crate::backends::{
common::{
- CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending,
+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
+ CommandBufferInitial, CommandBufferPending,
},
cpu::error::CpuError,
};
@@ -58,12 +58,12 @@ impl CommandBufferEncoding for CpuCommandBuffer {
fn encode_copy(
&mut self,
- src: &Box<[u8]>,
- dst: &mut Box<[u8]>,
+ src: &UnsafeCell<Box<[u8]>>,
+ dst: &mut UnsafeCell<Box<[u8]>>,
size: usize,
) {
- let src = src.as_ptr();
- let dst = dst.as_ptr() as *mut u8;
+ let src = unsafe { &*src.get() }.as_ptr();
+ let dst = dst.get_mut().as_mut_ptr();
self.push_command(Box::new(move || unsafe {
std::ptr::copy(src, dst, size);
}))
@@ -71,17 +71,24 @@ impl CommandBufferEncoding for CpuCommandBuffer {
fn encode_fill(
&mut self,
- dst: &mut Box<[u8]>,
+ dst: &mut UnsafeCell<Box<[u8]>>,
range: std::ops::Range<usize>,
value: u8,
) {
let size = range.end - range.start;
- let dst = dst[range].as_ptr() as *mut u8;
+ let dst = dst.get_mut()[range].as_ptr() as *mut u8;
self.push_command(Box::new(move || unsafe {
dst.write_bytes(value, size);
}))
}
+ fn encode_barrier(
+ &mut self,
+ _after: AccessFlags,
+ _before: AccessFlags,
+ ) {
+ }
+
fn encode_wait_for_event(
&mut self,
event: &RefCell<u64>,
diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rs
index 43e1c13b2..e0d5e314d 100644
--- a/crates/uzu/src/backends/cpu/context.rs
+++ b/crates/uzu/src/backends/cpu/context.rs
@@ -1,15 +1,23 @@
-use std::{cell::RefCell, path::Path, rc::Rc};
+use std::{
+ cell::{RefCell, UnsafeCell},
+ path::Path,
+ rc::Rc,
+};
use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};
-use crate::backends::common::Context;
+use crate::backends::common::{Allocation, AllocationPool, AllocationType, Allocator, Context};
-pub struct CpuContext;
+pub struct CpuContext {
+ allocator: Rc<Allocator<Cpu>>,
+}
impl Context for CpuContext {
type Backend = Cpu;
fn new() -> Result<Rc<Self>, CpuError> {
- Ok(Rc::new(CpuContext))
+ Ok(Rc::new_cyclic(|weak_self| CpuContext {
+ allocator: Allocator::new(weak_self.clone()),
+ }))
}
fn recommended_async_batch_size(
@@ -30,8 +38,23 @@ impl Context for CpuContext {
fn create_buffer(
&self,
size: usize,
- ) -> Result<Box<[u8]>, CpuError> {
- Ok(vec![0; size].into_boxed_slice())
+ ) -> Result<UnsafeCell<Box<[u8]>>, CpuError> {
+ Ok(UnsafeCell::new(vec![0; size].into_boxed_slice()))
+ }
+
+ fn create_allocation(
+ &self,
+ size: usize,
+ allocation_type: AllocationType<Cpu>,
+ ) -> Result<Allocation<Cpu>, CpuError> {
+ self.allocator.allocate(size, allocation_type)
+ }
+
+ fn create_allocation_pool(
+ &self,
+ reusable: bool,
+ ) -> AllocationPool<Cpu> {
+ self.allocator.create_pool(reusable)
}
fn create_command_buffer(&self) -> Result<CpuCommandBuffer, CpuError> {
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 0e9cb49d5..3991c4f8a 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -3,8 +3,11 @@ use half::{bf16, f16};
use crate::{
DataType,
backends::{
- common::kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},
- cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext},
+ common::{
+ Encoder,
+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},
+ },
+ cpu::{Cpu, context::CpuContext},
},
};
@@ -31,8 +34,9 @@ impl MatmulKernel for MatmulCpuKernel {
&mut self,
_context: &CpuContext,
arguments: MatmulArguments<Cpu>,
- command_buffer: &mut CpuCommandBuffer,
+ encoder: &mut Encoder<Cpu>,
) {
+ let command_buffer = encoder.as_command_buffer_mut();
let m = arguments.batch as usize;
let n = arguments.output_dim as usize;
let k = arguments.input_dim as usize;
@@ -42,10 +46,10 @@ impl MatmulKernel for MatmulCpuKernel {
let a_offset = arguments.a_offset as usize;
let data_type = self.data_type;
- let a_ptr = arguments.a.as_ptr().wrapping_byte_add(a_offset);
- let b_ptr = arguments.b.as_ptr();
- let d_ptr = arguments.d.as_mut_ptr();
- let bias_ptr = arguments.bias.map(|b| b.as_ptr());
+ let a_ptr = unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset);
+ let b_ptr = unsafe { &*arguments.b.get() }.as_ptr();
+ let d_ptr = arguments.d.get_mut().as_mut_ptr();
+ let bias_ptr = arguments.bias.map(|b| unsafe { &*b.get() }.as_ptr());
command_buffer.push_command(move || match data_type {
DataType::F32 => {
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index 97b583bda..bd242efb9 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -15,6 +15,7 @@ impl Backend for Metal {
type Kernels = MetalKernels;
type Error = MetalError;
+ const MIN_ALLOCATION_ALIGNMENT: usize = 64;
// Metal's set_bytes supports up to 4KB per bound value.
// https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc
const MAX_INLINE_BYTES: usize = 4096;
diff --git a/crates/uzu/src/backends/metal/buffer.rs b/crates/uzu/src/backends/metal/buffer.rs
index 84c0a4914..85d4fe4de 100644
--- a/crates/uzu/src/backends/metal/buffer.rs
+++ b/crates/uzu/src/backends/metal/buffer.rs
@@ -20,11 +20,11 @@ impl Buffer for Retained<ProtocolObject<dyn MTLBuffer>> {
self.contents()
}
- fn length(&self) -> usize {
- (**self).length()
+ fn gpu_ptr(&self) -> usize {
+ self.gpu_address() as usize
}
- fn id(&self) -> usize {
- Retained::as_ptr(self) as usize
+ fn length(&self) -> usize {
+ (**self).length()
}
}
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index e27990986..894b5eb96 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -9,8 +9,8 @@ use objc2::{Message, rc::Retained, runtime::ProtocolObject};
use super::Metal;
use crate::backends::{
common::{
- CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending,
+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
+ CommandBufferInitial, CommandBufferPending,
},
metal::error::MetalError,
};
@@ -141,6 +141,13 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
self.ensure_blit().fill_buffer_range_value(dst, range, value);
}
+ fn encode_barrier(
+ &mut self,
+ _after: AccessFlags,
+ _before: AccessFlags,
+ ) {
+ }
+
fn encode_wait_for_event(
&mut self,
event: &Retained<ProtocolObject<dyn MTLEvent>>,
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index de0547a48..30db27a09 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -15,13 +15,17 @@ use super::{
metal_extensions::{DeviceGeneration, LibraryPipelineExtensions},
};
use crate::{
- backends::{common::Context, metal::command_buffer::MetalCommandBufferInitial},
+ backends::{
+ common::{Allocation, AllocationPool, AllocationType, Allocator, Context},
+ metal::command_buffer::MetalCommandBufferInitial,
+ },
utils::ModelSize,
};
pub struct MetalContext {
pub device: Retained<ProtocolObject<dyn MTLDevice>>,
pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,
+ allocator: Rc<Allocator<Metal>>,
device_capabilities: MetalDeviceCapabilities,
library: Retained<ProtocolObject<dyn MTLLibrary>>,
pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
@@ -66,9 +70,10 @@ impl Context for MetalContext {
let device_capabilities = MetalDeviceCapabilities::from_device(&device);
- Ok(Rc::new(Self {
+ Ok(Rc::new_cyclic(|weak_self| Self {
device,
command_queue,
+ allocator: Allocator::new(weak_self.clone()),
device_capabilities,
library,
pipeline_cache: RefCell::new(HashMap::new()),
@@ -107,6 +112,21 @@ impl Context for MetalContext {
self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).ok_or(MetalError::CannotCreateBuffer)
}
+ fn create_allocation(
+ &self,
+ size: usize,
+ allocation_type: AllocationType<Metal>,
+ ) -> Result<Allocation<Metal>, MetalError> {
+ self.allocator.allocate(size, allocation_type)
+ }
+
+ fn create_allocation_pool(
+ &self,
+ reusable: bool,
+ ) -> AllocationPool<Metal> {
+ self.allocator.create_pool(reusable)
+ }
+
fn create_command_buffer(&self) -> Result<MetalCommandBufferInitial, MetalError> {
Ok(MetalCommandBufferInitial::new(
self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs
index f0dada6f4..ae1d4d400 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs
@@ -11,13 +11,14 @@ use crate::{
DataType,
backends::{
common::{
+ Encoder,
gpu_types::GemmParams,
kernel::{
TensorAddBiasKernel,
matmul::{MatmulArguments, MatmulError, MatmulKernel},
},
},
- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext},
+ metal::{Metal, context::MetalContext},
},
};
@@ -109,7 +110,7 @@ impl MatmulMetalKernel {
fn encode_gemv(
&mut self,
context: &MetalContext,
- command_buffer: &mut MetalCommandBufferEncoding,
+ encoder: &mut Encoder<Metal>,
arguments: MatmulArguments<Metal>,
) -> Result<(), MatmulError<Metal>> {
let m = arguments.batch;
@@ -182,7 +183,7 @@ impl MatmulMetalKernel {
bias_stride,
arguments.batch,
specialization.output_rows_per_threadgroup() as i32,
- command_buffer,
+ encoder,
);
let bias_is_fused = apply_output_scale_and_accumulate && has_bias;
@@ -193,7 +194,7 @@ impl MatmulMetalKernel {
arguments.d,
arguments.batch as usize,
arguments.output_dim as usize,
- command_buffer,
+ encoder,
)?;
}
@@ -203,7 +204,7 @@ impl MatmulMetalKernel {
fn encode_gemm(
&mut self,
context: &MetalContext,
- command_buffer: &mut MetalCommandBufferEncoding,
+ encoder: &mut Encoder<Metal>,
arguments: MatmulArguments<Metal>,
) -> Result<(), MatmulError<Metal>> {
let m = arguments.batch;
@@ -247,7 +248,7 @@ impl MatmulMetalKernel {
std::slice::from_ref(&params),
group_count_x,
group_count_y,
- command_buffer,
+ encoder,
);
self.encode_bias_add(
@@ -256,7 +257,7 @@ impl MatmulMetalKernel {
arguments.d,
arguments.batch as usize,
arguments.output_dim as usize,
- command_buffer,
+ encoder,
)?;
Ok(())
@@ -269,7 +270,7 @@ impl MatmulMetalKernel {
output: &mut <Metal as crate::backends::common::Backend>::Buffer,
batch: usize,
output_dim: usize,
- command_buffer: &mut MetalCommandBufferEncoding,
+ encoder: &mut Encoder<Metal>,
) -> Result<(), MatmulError<Metal>> {
let Some(bias) = bias else {
return Ok(());
@@ -289,7 +290,7 @@ impl MatmulMetalKernel {
output,
output_dim as u32,
total_length as u32,
- command_buffer,
+ encoder,
);
Ok(())
}
@@ -327,12 +328,12 @@ impl MatmulKernel for MatmulMetalKernel {
&mut self,
context: &MetalContext,
arguments: MatmulArguments<Metal>,
- command_buffer: &mut MetalCommandBufferEncoding,
+ encoder: &mut Encoder<Metal>,
) {
if Self::is_gemv_eligible(&arguments) {
- self.encode_gemv(context, command_buffer, arguments).expect("Failed to encode GEMV kernel");
+ self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");
} else {
- self.encode_gemm(context, command_buffer, arguments).expect("Failed to encode GEMM kernel");
+ self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index 93badc6a7..181d9afa0 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -152,4 +152,4 @@ PUBLIC KERNEL(MoeRouterTopK)(
out_probs[i] = static_cast<ScalarT>(0.0f);
}
}
-}
\ No newline at end of file
+}
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 1b16fe499..c43a205aa 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -7,9 +7,7 @@ use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
DataType,
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- },
+ backends::common::{Backend, Encoder},
encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
@@ -108,54 +106,32 @@ impl<B: Backend> Classifier<B> {
let encoding_params = EncodingParameters::new();
- let mut command_buffer = self
- .context
- .context
- .create_command_buffer()
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
- .start_encoding();
-
- self.context
- .embed
- .encode_lookup(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .embedding_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
+
+ self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());
+ state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().embedding_norm().clone());
}
for layer in self.context.layers.iter() {
- layer
- .encode(&mut state, &encoding_params, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
- self.context
- .output_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());
+ state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().output_norm.clone());
}
- self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .prediction_head
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
let logits = self.copy_logits_from_state(&state)?;
@@ -183,41 +159,19 @@ impl<B: Backend> Classifier<B> {
let encoding_params = EncodingParameters::new();
- let mut command_buffer = self
- .context
- .context
- .create_command_buffer()
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
- .start_encoding();
-
- self.context
- .embed
- .encode_lookup(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .embedding_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
+
+ self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
for layer in self.context.layers.iter() {
- layer
- .encode(&mut state, &encoding_params, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
- self.context
- .output_norm
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .prediction_head
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- command_buffer
- .end_encoding()
- .submit()
- .wait_until_completed()
- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
+ self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+
+ encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
let logits = self.copy_logits_from_state(&state)?;
diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rs
index 810b64344..0bc689869 100644
--- a/crates/uzu/src/encodable_block/activation.rs
+++ b/crates/uzu/src/encodable_block/activation.rs
@@ -5,7 +5,7 @@ use std::ops::DerefMut;
use crate::{
DataType,
backends::common::{
- ActivationConfig, Backend, CommandBuffer,
+ ActivationConfig, Backend, Encoder,
gpu_types::ActivationType,
kernel::{ActivationKernel, Kernels},
},
@@ -40,7 +40,7 @@ impl<B: Backend> Activation<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let input_array = arrays[0].borrow();
@@ -58,7 +58,7 @@ impl<B: Backend> Activation<B> {
output_array.buffer().borrow_mut().deref_mut(),
n as u32,
self.activation.act_type(),
- command_buffer,
+ encoder,
);
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index c5dad382e..a66bbf4da 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -8,7 +8,7 @@ use std::{
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,
SigmoidGateKernel,
@@ -147,7 +147,7 @@ impl<B: Backend> Attention<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
@@ -258,7 +258,7 @@ impl<B: Backend> Attention<B> {
suffix_length as u32,
0u32,
max_sequence_length as u32,
- command_buffer,
+ encoder,
);
}
@@ -318,7 +318,7 @@ impl<B: Backend> Attention<B> {
suffix_length as u32,
segment_prefix_length as u32,
max_sequence_length as u32,
- command_buffer,
+ encoder,
);
}
@@ -371,9 +371,7 @@ impl<B: Backend> Attention<B> {
is_causal: self.is_causal,
scale,
};
- self.gemm_block
- .encode(state.context(), command_buffer, args)
- .expect("Failed to encode AttentionGemmBlock");
+ self.gemm_block.encode(state.context(), encoder, args).expect("Failed to encode AttentionGemmBlock");
},
KernelVariant::SinglePass => {
let kernel = match self.single_pass_kernels.get(&kernel_key) {
@@ -399,7 +397,7 @@ impl<B: Backend> Attention<B> {
sinks_buffer,
num_heads as u32,
suffix_length as u32,
- command_buffer,
+ encoder,
)
},
KernelVariant::TwoPass => {
@@ -432,7 +430,7 @@ impl<B: Backend> Attention<B> {
mask_q_seq_stride_opt,
mask_head_stride_opt,
sinks_buffer,
- command_buffer,
+ encoder,
);
kernel_pass2.encode(
partials_buf_borrow.deref(),
@@ -441,7 +439,7 @@ impl<B: Backend> Attention<B> {
attention_output_buf_borrow.deref_mut(),
num_heads as u32,
suffix_length as u32,
- command_buffer,
+ encoder,
);
},
}
@@ -455,7 +453,7 @@ impl<B: Backend> Attention<B> {
gate_buf_borrow.deref(),
attention_output_buf_borrow.deref_mut(),
total_elements,
- command_buffer,
+ encoder,
);
}
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index 5ead1cf31..a2344e829 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -2,7 +2,7 @@ use std::rc::Rc;
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, Encoder},
config::TransformerLayerConfig,
encodable_block::{
Attention, EncodingParameters, Linear, Mlp, Normalization, QKNorm, Rope, TensorAddSwap, TensorCopy,
@@ -225,95 +225,83 @@ impl<B: Backend> ClassifierLayer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
#[cfg(feature = "tracing")]
let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());
}
- self.copy_main_to_shortcut_mixer.encode(state, command_buffer)?;
+ self.copy_main_to_shortcut_mixer.encode(state, encoder)?;
if let Some(ref pre_attn_norm) = self.pre_attention_norm {
- pre_attn_norm.encode(state, command_buffer)?;
+ pre_attn_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().pre_attention_norm.clone(),
- );
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
}
} else {
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().pre_attention_norm.clone(),
- );
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
}
}
- self.qkv_projection.encode(state, command_buffer)?;
+ self.qkv_projection.encode(state, encoder)?;
if let Some(ref qk_norm) = self.qk_norm {
- qk_norm.encode(state, command_buffer)?;
+ qk_norm.encode(state, encoder)?;
}
- self.rope.encode(state, command_buffer)?;
- self.attention.encode(state, parameters, command_buffer)?;
- self.out_projection.encode(state, command_buffer)?;
+ self.rope.encode(state, encoder)?;
+ self.attention.encode(state, parameters, encoder)?;
+ self.out_projection.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
}
if let Some(ref post_attn_norm) = self.post_attention_norm {
- post_attn_norm.encode(state, command_buffer)?;
+ post_attn_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().post_attention_norm.clone(),
- );
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());
}
}
- self.mixer_residual_add.encode(state, command_buffer)?;
+ self.mixer_residual_add.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
}
- self.copy_main_to_shortcut_mlp.encode(state, command_buffer)?;
+ self.copy_main_to_shortcut_mlp.encode(state, encoder)?;
- self.pre_mlp_norm.encode(state, command_buffer)?;
+ self.pre_mlp_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
}
- self.mlp.encode(state, command_buffer)?;
+ self.mlp.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());
}
if let Some(ref post_mlp_norm) = self.post_mlp_norm {
- post_mlp_norm.encode(state, command_buffer)?;
+ post_mlp_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
}
}
- self.mlp_residual_add.encode(state, command_buffer)?;
+ self.mlp_residual_add.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());
}
Ok(())
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index c60e342b2..901a67b12 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -6,7 +6,7 @@ use thiserror::Error;
use crate::{
DataType, DecoderConfig,
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, Encoder},
config::{DecoderLayerType, MixerConfig},
encodable_block::{Embedding, EncodingParameters, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError},
forward_pass::{
@@ -166,30 +166,30 @@ impl<B: Backend> Decoder<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), DecoderError<B>> {
- self.embed.encode_lookup(state, command_buffer)?;
+ self.embed.encode_lookup(state, encoder)?;
for layer in self.layers.iter() {
- layer.encode(state, parameters, command_buffer).map_err(DecoderError::BackendError)?;
+ layer.encode(state, parameters, encoder).map_err(DecoderError::BackendError)?;
}
if state.is_prefilling() {
return Ok(());
}
- self.norm.encode(state, command_buffer).map_err(DecoderError::BackendError)?;
+ self.norm.encode(state, encoder).map_err(DecoderError::BackendError)?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, traces.borrow().output_norm.clone());
}
- self.embed.encode_readout(state, command_buffer)?;
+ self.embed.encode_readout(state, encoder)?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(command_buffer, ArrayId::Logits, traces.borrow().logits.clone());
+ state.encode_copy_array(encoder, ArrayId::Logits, traces.borrow().logits.clone());
}
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index 775dea92f..f58fabf51 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -8,7 +8,7 @@ use thiserror::Error;
use crate::{
DataType, EmbeddingConfig,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
kernel::{
FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,
matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
@@ -421,7 +421,7 @@ impl<B: Backend> Embedding<B> {
pub fn encode_lookup(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), EmbeddingError<B>> {
let batch_size = state.active_suffix_length() as u32;
@@ -461,7 +461,7 @@ impl<B: Backend> Embedding<B> {
self.vocab_size,
self.model_dim,
self.input_scale,
- command_buffer,
+ encoder,
),
EmbeddingTying::Tied {
ty:
@@ -493,7 +493,7 @@ impl<B: Backend> Embedding<B> {
self.vocab_size,
self.model_dim,
self.input_scale,
- command_buffer,
+ encoder,
);
},
};
@@ -504,7 +504,7 @@ impl<B: Backend> Embedding<B> {
pub fn encode_readout(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), EmbeddingError<B>> {
let batch_size = state.sampling_length();
@@ -561,7 +561,7 @@ impl<B: Backend> Embedding<B> {
leading_dimension_d: output_dim as i32,
transpose_b: true,
},
- command_buffer,
+ encoder,
);
},
EmbeddingTying::Tied {
@@ -585,7 +585,7 @@ impl<B: Backend> Embedding<B> {
},
} => {
readout.encode(
- command_buffer,
+ encoder,
QuantizedMatmulArguments {
a_buffer: input,
a_offset: input_offset,
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index f785afb80..d0a55a4d9 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -5,7 +5,7 @@ use std::rc::Rc;
use super::MixerExecutables;
use crate::{
DataType, DecoderLayerConfig,
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, Encoder},
config::{DecoderLayerType, MixerConfig},
encodable_block::{
Attention, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer, TensorAddSwap,
@@ -262,23 +262,23 @@ impl<B: Backend> LayerExecutables<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
#[cfg(feature = "tracing")]
let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());
}
- self.copy_main_to_shortcut.encode(state, command_buffer)?;
+ self.copy_main_to_shortcut.encode(state, encoder)?;
// shortcut = input
- self.pre_attention_norm.encode(state, command_buffer)?;
+ self.pre_attention_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
}
match &self.mixer {
@@ -290,89 +290,85 @@ impl<B: Backend> LayerExecutables<B> {
attention,
out_projection,
} => {
- qkv_projection.encode(state, command_buffer)?;
+ qkv_projection.encode(state, encoder)?;
if let Some(gate_proj) = gate_projection {
- gate_proj.encode(state, command_buffer)?;
+ gate_proj.encode(state, encoder)?;
}
if let Some(norm) = qk_norm {
- norm.encode(state, command_buffer)?;
+ norm.encode(state, encoder)?;
}
- rope.encode(state, command_buffer)?;
- attention.encode(state, parameters, command_buffer)?;
- out_projection.encode(state, command_buffer)?;
+ rope.encode(state, encoder)?;
+ attention.encode(state, parameters, encoder)?;
+ out_projection.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
}
},
MixerExecutables::StateSpace {
mixer,
} => {
- mixer.encode(state, command_buffer)?;
+ mixer.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
}
},
MixerExecutables::ShortConv {
mixer,
} => {
- mixer.encode(state, command_buffer)?;
+ mixer.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());
}
},
}
if let Some(post_attention_norm) = &self.post_attention_norm {
- post_attention_norm.encode(state, command_buffer)?;
+ post_attention_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(
- command_buffer,
- ArrayId::Main,
- layer_traces.borrow().post_attention_norm.clone(),
- );
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());
}
}
//main = attention_result
- self.main_shortcut_add_swap.encode(state, command_buffer)?;
+ self.main_shortcut_add_swap.encode(state, encoder)?;
// shortcut = input + attention_result
// main = input + attention_result
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
}
- self.pre_mlp_norm.encode(state, command_buffer)?;
+ self.pre_mlp_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
}
- self.mlp.encode(state, command_buffer)?;
+ self.mlp.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());
}
if let Some(post_mlp_norm) = &self.post_mlp_norm {
- post_mlp_norm.encode(state, command_buffer)?;
+ post_mlp_norm.encode(state, encoder)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
}
}
// main = mlp_result
- self.main_shortcut_add_swap.encode(state, command_buffer)?;
+ self.main_shortcut_add_swap.encode(state, encoder)?;
// shortcut = input + attention_result + mlp_result
// main = input + attention_result + mlp_result
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());
+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());
}
Ok(())
diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rs
index 301baf9e6..86ed51fe0 100644
--- a/crates/uzu/src/encodable_block/layer_norm.rs
+++ b/crates/uzu/src/encodable_block/layer_norm.rs
@@ -11,7 +11,7 @@ use thiserror::Error;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, LayerNormKernel},
},
config::{NormalizationConfig, UpcastMode},
@@ -71,7 +71,7 @@ impl<B: Backend> LayerNorm<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
@@ -103,7 +103,7 @@ impl<B: Backend> LayerNorm<B> {
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
full_layer,
- command_buffer,
+ encoder,
);
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rs
index e54049ec7..3e4e59cbf 100644
--- a/crates/uzu/src/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs
@@ -10,7 +10,7 @@ use super::Linear;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -129,7 +129,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
@@ -153,7 +153,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
leading_dimension_d: self.output_dim as i32,
transpose_b: true,
},
- command_buffer,
+ encoder,
);
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rs
index faaf316ab..7d8478fc6 100644
--- a/crates/uzu/src/encodable_block/linear/mod.rs
+++ b/crates/uzu/src/encodable_block/linear/mod.rs
@@ -6,7 +6,7 @@ pub use quantized::{QuantizedLinear, QuantizedLinearError};
use thiserror::Error;
use crate::{
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, Encoder},
config::LinearConfig,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -16,7 +16,7 @@ pub trait Linear<B: Backend> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error>;
}
diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rs
index 3f46ab886..ebcfcd6a7 100644
--- a/crates/uzu/src/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/encodable_block/linear/quantized.rs
@@ -10,7 +10,7 @@ use super::Linear;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{
Kernels, TensorAddBiasKernel,
quant_matmul::{
@@ -249,7 +249,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
@@ -260,7 +260,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
self.kernel
.encode(
- command_buffer,
+ encoder,
QuantizedMatmulArguments {
a_buffer: input_array.buffer().borrow().deref(),
a_offset: 0,
@@ -284,7 +284,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
output_buf_borrow.deref_mut(),
self.output_dim as u32,
total_length as u32,
- command_buffer,
+ encoder,
);
}
Ok(())
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index bc0d2ade9..035393bb8 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -9,7 +9,7 @@ use crate::{
DataType,
array::Array,
backends::common::{
- Backend, CommandBuffer, Kernels,
+ Backend, Encoder, Kernels,
kernel::{
Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,
ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
@@ -146,7 +146,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_split_inproj(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -179,14 +179,14 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
inner_dim as u32,
num_heads as u32,
- command_buffer,
+ encoder,
)
}
fn run_conv_scan(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -231,7 +231,7 @@ impl<B: Backend> MambaMixer<B> {
inner_dim as u32,
proj_dim as u32,
self.config.activation.act_type(),
- command_buffer,
+ encoder,
);
}
} else {
@@ -247,7 +247,7 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
suffix_length as u32,
conv_dim as u32,
- command_buffer,
+ encoder,
);
Some(buffer)
@@ -276,7 +276,7 @@ impl<B: Backend> MambaMixer<B> {
inner_dim as u32,
proj_dim as u32,
self.config.activation.act_type(),
- command_buffer,
+ encoder,
)
}
}
@@ -285,7 +285,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_prefill_ssm(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
suffix_length: usize,
) {
let base_arrays = state.arrays(&[
@@ -309,7 +309,7 @@ impl<B: Backend> MambaMixer<B> {
let skip_rc = self.skip_connection_weight.buffer();
let skip_borrow = skip_rc.borrow();
self.ssd_prefill.encode(
- command_buffer,
+ encoder,
SSDPrefillArguments {
x: x.buffer().borrow().deref(),
dt: dt.buffer().borrow().deref(),
@@ -336,7 +336,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_decode_ssm(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -390,29 +390,29 @@ impl<B: Backend> MambaMixer<B> {
suffix_length as u32,
h,
dh,
- command_buffer,
+ encoder,
);
}
pub(crate) fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
return Ok(());
}
- self.in_projection.encode(state, command_buffer)?;
- self.run_split_inproj(state, command_buffer, active_suffix_length);
- self.run_conv_scan(state, command_buffer, active_suffix_length);
+ self.in_projection.encode(state, encoder)?;
+ self.run_split_inproj(state, encoder, active_suffix_length);
+ self.run_conv_scan(state, encoder, active_suffix_length);
if active_suffix_length == 1 {
- self.run_decode_ssm(state, command_buffer, active_suffix_length);
+ self.run_decode_ssm(state, encoder, active_suffix_length);
} else {
- self.run_prefill_ssm(state, command_buffer, active_suffix_length);
+ self.run_prefill_ssm(state, encoder, active_suffix_length);
}
- self.out_projection.encode(state, command_buffer)?;
+ self.out_projection.encode(state, encoder)?;
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/mlp/dense.rs b/crates/uzu/src/encodable_block/mlp/dense.rs
index 1c2e9eac1..eaed083d8 100644
--- a/crates/uzu/src/encodable_block/mlp/dense.rs
+++ b/crates/uzu/src/encodable_block/mlp/dense.rs
@@ -4,7 +4,7 @@ use std::ops::{Deref, DerefMut};
use super::{super::linear::Linear, Mlp};
use crate::{
- backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
+ backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
forward_pass::state::{ArrayId, ForwardPassState},
};
@@ -32,10 +32,10 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
// Up
- self.up.encode(state, command_buffer)?;
+ self.up.encode(state, encoder)?;
// Gate act+mul (fused_up -> hidden)
let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
@@ -47,7 +47,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
let hidden_buf_rc = hidden.buffer();
let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();
self.gate
- .encode(command_buffer, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)
+ .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)
.expect("Failed to encode MLP activation/mul kernel");
drop(hidden_buf_borrow);
drop(fused_buf_borrow);
@@ -55,7 +55,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
drop(hidden);
// Down
- self.down.encode(state, command_buffer)?;
+ self.down.encode(state, encoder)?;
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/mlp/mod.rs b/crates/uzu/src/encodable_block/mlp/mod.rs
index e6d28a089..23cb57405 100644
--- a/crates/uzu/src/encodable_block/mlp/mod.rs
+++ b/crates/uzu/src/encodable_block/mlp/mod.rs
@@ -8,7 +8,7 @@ use thiserror::Error;
use super::linear::{Linear, LinearBlockError};
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
+ backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
config::MLPConfig,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
@@ -18,7 +18,7 @@ pub trait Mlp<B: Backend> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error>;
}
diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
index 4339670ef..62b2d1bb1 100644
--- a/crates/uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -12,7 +12,7 @@ use crate::{
DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,
array::Array,
backends::common::{
- Backend, CommandBuffer, CommandBufferEncoding, Kernels,
+ Backend, Encoder, Kernels,
gpu_types::ActivationType,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
@@ -193,7 +193,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let suffix_length = state.active_suffix_length();
let arrays = state.arrays(&[
@@ -256,28 +256,28 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
// Clear topk_ids and tok2row buffers
if topk_bytes > 0 {
- command_buffer.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);
+ encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);
}
if tok2row_bytes > 0 {
- command_buffer.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);
+ encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);
}
// Clear hidden buffer
let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
if hidden_bytes > 0 {
- command_buffer.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);
+ encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);
}
// Clear y_partial buffer
let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
if y_partial_bytes > 0 {
- command_buffer.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);
+ encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);
}
// Clear x_perm buffer
let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
if x_perm_bytes > 0 {
- command_buffer.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);
+ encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);
}
}
@@ -305,7 +305,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
e as u32,
k as u32,
self.router_renorm,
- command_buffer,
+ encoder,
);
}
@@ -317,7 +317,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
suffix_length as u32,
e as u32,
k as u32,
- command_buffer,
+ encoder,
);
let num_blocks = ((suffix_length + 255) / 256).max(1);
@@ -330,7 +330,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
num_blocks as u32,
num_tiles as u32,
0u32,
- command_buffer,
+ encoder,
);
self.scatter_map_kernel.encode(
topk_ids_buf.borrow().deref(),
@@ -346,11 +346,11 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
num_blocks as u32,
num_tiles as u32,
tok2row_buf.borrow_mut().deref_mut(),
- command_buffer,
+ encoder,
);
self.gather_kernels.encode(
- command_buffer,
+ encoder,
self.data_type,
MoeGatherArguments {
x_buffer: main_buf.borrow().deref(),
@@ -377,7 +377,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
self.experts_two_pass_decode_kernel.encode(
- command_buffer,
+ encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: x_perm_buf.borrow().deref(),
expert_offsets: offsets_buf.borrow().deref(),
@@ -454,7 +454,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
silu_alpha,
data_type: self.data_type,
};
- self.experts_two_pass_prefill_kernel.encode(command_buffer, args);
+ self.experts_two_pass_prefill_kernel.encode(encoder, args);
}
self.finalize_kernel.encode(
@@ -465,7 +465,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
suffix_length as u32,
self.model_dim as u32,
k as u32,
- command_buffer,
+ encoder,
);
Ok(())
diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rs
index 53d5cfd51..7c99d0873 100644
--- a/crates/uzu/src/encodable_block/normalization.rs
+++ b/crates/uzu/src/encodable_block/normalization.rs
@@ -5,7 +5,7 @@ use thiserror::Error;
use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, Encoder},
config::NormalizationConfig,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -60,11 +60,11 @@ impl<B: Backend> Normalization<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
match self {
- Self::LayerNorm(layer_norm) => layer_norm.encode(state, command_buffer),
- Self::RMSNorm(rms_norm) => rms_norm.encode(state, command_buffer),
+ Self::LayerNorm(layer_norm) => layer_norm.encode(state, encoder),
+ Self::RMSNorm(rms_norm) => rms_norm.encode(state, encoder),
}
}
}
diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rs
index b6cfba724..74903ab54 100644
--- a/crates/uzu/src/encodable_block/pooling.rs
+++ b/crates/uzu/src/encodable_block/pooling.rs
@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, PoolingClsKernel, PoolingMeanKernel},
},
config::PoolingType,
@@ -25,11 +25,11 @@ impl<B: Backend> PoolingKernel<B> {
seq_len: u32,
hidden_dim: u32,
batch_size: u32,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) {
match self {
- Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),
- Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),
+ Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
+ Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
}
}
}
@@ -61,7 +61,7 @@ impl<B: Backend> Pooling<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let batch_size = 1;
let seq_len = state.aux_buffers_suffix_length();
@@ -76,13 +76,13 @@ impl<B: Backend> Pooling<B> {
seq_len as u32,
self.model_dim as u32,
batch_size,
- command_buffer,
+ encoder,
);
#[cfg(feature = "tracing")]
{
let output_pooling_trace = state.traces().borrow().output_pooling().clone();
- state.encode_copy_array(command_buffer, ArrayId::ClassifierPooling, output_pooling_trace);
+ state.encode_copy_array(encoder, ArrayId::ClassifierPooling, output_pooling_trace);
}
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rs
index 62e62ba44..aca2827c7 100644
--- a/crates/uzu/src/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/encodable_block/prediction_head.rs
@@ -3,7 +3,7 @@
#[cfg(feature = "tracing")]
use crate::forward_pass::state::ArrayId;
use crate::{
- backends::common::{Backend, CommandBuffer},
+ backends::common::{Backend, Encoder},
encodable_block::{Activation, Normalization, linear::Linear},
forward_pass::state::ForwardPassState,
};
@@ -37,21 +37,17 @@ impl<B: Backend> ClassifierPredictionHead<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- self.dense.encode(state, command_buffer)?;
- self.activation.encode(state, command_buffer)?;
- self.norm.encode(state, command_buffer)?;
- self.readout.encode(state, command_buffer)?;
+ self.dense.encode(state, encoder)?;
+ self.activation.encode(state, encoder)?;
+ self.norm.encode(state, encoder)?;
+ self.readout.encode(state, encoder)?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(
- command_buffer,
- ArrayId::ClassifierPredictionHeadLogits,
- traces.borrow().logits.clone(),
- );
+ state.encode_copy_array(encoder, ArrayId::ClassifierPredictionHeadLogits, traces.borrow().logits.clone());
}
Ok(())
diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
index 6c5a7cdab..9ec1ebf44 100644
--- a/crates/uzu/src/encodable_block/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -11,7 +11,7 @@ use thiserror::Error;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, QKNormKernel},
},
config::{NormalizationConfig, UpcastMode},
@@ -124,7 +124,7 @@ impl<B: Backend> QKNorm<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
let qkv_shape = {
@@ -152,7 +152,7 @@ impl<B: Backend> QKNorm<B> {
0,
self.num_q_heads as u32,
query_config.upcast_mode == UpcastMode::FullLayer,
- command_buffer,
+ encoder,
);
}
@@ -173,7 +173,7 @@ impl<B: Backend> QKNorm<B> {
self.num_q_heads as u32,
self.num_kv_heads as u32,
key_config.upcast_mode == UpcastMode::FullLayer,
- command_buffer,
+ encoder,
);
}
Ok(())
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 30ecb88f1..3745391e4 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -11,7 +11,7 @@ use thiserror::Error;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, RMSNormKernel},
},
config::{NormalizationConfig, UpcastMode},
@@ -85,7 +85,7 @@ impl<B: Backend> RMSNorm<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
@@ -130,7 +130,7 @@ impl<B: Backend> RMSNorm<B> {
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
self.config.upcast_mode == UpcastMode::FullLayer,
- command_buffer,
+ encoder,
);
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rs
index edee08bcd..8b23c8e99 100644
--- a/crates/uzu/src/encodable_block/rope.rs
+++ b/crates/uzu/src/encodable_block/rope.rs
@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, RopeKernel},
},
forward_pass::state::{ArrayId, ForwardPassState, RopeType},
@@ -31,7 +31,7 @@ impl<B: Backend> Rope<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let (suffix_length, num_heads, head_dim, num_groups, rope_dim, rope_max_seq_len) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
@@ -89,7 +89,7 @@ impl<B: Backend> Rope<B> {
num_groups as u32,
suffix_length as u32,
rope_max_seq_len as u32,
- command_buffer,
+ encoder,
);
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index 8b174add0..74c15aa24 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::sampling::{SamplingError, SamplingKernel},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -31,7 +31,7 @@ impl<B: Backend> Sampling<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");
@@ -77,7 +77,7 @@ impl<B: Backend> Sampling<B> {
sampling_method,
batch_size,
vocab_size,
- command_buffer,
+ encoder,
) {
panic!("Sampling encoding failed: {:?}", e);
}
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index 98406a2e7..d98a6b322 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -4,7 +4,7 @@ use crate::{
DataType,
array::Array,
backends::common::{
- Backend, CommandBuffer, Context, Kernels,
+ Backend, Context, Encoder, Kernels,
kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
},
config::{DecoderLayerType, ShortConvConfig},
@@ -130,19 +130,19 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
active_suffix_length: usize,
) {
self.clear_suffix_state_valid_range(state);
if active_suffix_length == 1 {
- self.run_decode_conv(state, compute, 1);
+ self.run_decode_conv(state, encoder, 1);
return;
}
let sampling_len = state.sampling_length();
if sampling_len == 0 {
- self.run_prefill_conv(state, compute, active_suffix_length);
+ self.run_prefill_conv(state, encoder, active_suffix_length);
return;
}
@@ -150,26 +150,26 @@ impl<B: Backend> ShortConvMixer<B> {
let trie_len = sampling_len;
if trie_len <= 1 {
- self.run_prefill_conv(state, compute, active_suffix_length);
+ self.run_prefill_conv(state, encoder, active_suffix_length);
return;
}
if sampling_start > 0 {
if sampling_start == 1 {
- self.run_decode_conv(state, compute, 1);
+ self.run_decode_conv(state, encoder, 1);
} else {
- self.run_prefill_conv(state, compute, sampling_start);
+ self.run_prefill_conv(state, encoder, sampling_start);
}
}
- self.run_trie_conv(state, compute, sampling_start, trie_len);
+ self.run_trie_conv(state, encoder, sampling_start, trie_len);
self.set_suffix_state_valid_range(state, sampling_start, trie_len);
}
fn run_prefill_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -204,7 +204,7 @@ impl<B: Backend> ShortConvMixer<B> {
suffix_length as u32,
self.model_dim as u32 * 3,
self.model_dim as u32,
- compute,
+ encoder,
);
self.short_conv_prefill.encode(
@@ -219,14 +219,14 @@ impl<B: Backend> ShortConvMixer<B> {
self.model_dim as u32 * 3,
state_stride as u32,
self.model_dim as u32,
- compute,
+ encoder,
)
}
fn run_trie_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
sampling_start: usize,
trie_len: usize,
) {
@@ -278,14 +278,14 @@ impl<B: Backend> ShortConvMixer<B> {
in_proj_stride as u32,
state_stride as u32,
self.model_dim as u32,
- compute,
+ encoder,
);
}
fn run_decode_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -317,25 +317,25 @@ impl<B: Backend> ShortConvMixer<B> {
self.model_dim as u32 * 3,
state_stride as u32,
self.model_dim as u32,
- compute,
+ encoder,
)
}
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
return Ok(());
}
- self.in_projection.encode(state, command_buffer)?;
+ self.in_projection.encode(state, encoder)?;
- self.run_conv(state, command_buffer, active_suffix_length);
+ self.run_conv(state, encoder, active_suffix_length);
- self.out_projection.encode(state, command_buffer)?;
+ self.out_projection.encode(state, encoder)?;
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rs
index ea0b2b9d1..05db52008 100644
--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs
@@ -5,7 +5,7 @@ use std::ops::DerefMut;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, TensorAddSwapKernel},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -32,7 +32,7 @@ impl<B: Backend> TensorAddSwap<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
@@ -46,7 +46,7 @@ impl<B: Backend> TensorAddSwap<B> {
skip_array.buffer().borrow_mut().deref_mut(),
main_array.buffer().borrow_mut().deref_mut(),
length as u32,
- command_buffer,
+ encoder,
);
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rs
index 74af8ac92..00f0ec5af 100644
--- a/crates/uzu/src/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/encodable_block/tensor_copy.rs
@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::{Kernels, TensorCopyKernel},
},
forward_pass::state::{ArrayId, ForwardPassState},
@@ -32,7 +32,7 @@ impl<B: Backend> TensorCopy<B> {
pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
@@ -46,7 +46,7 @@ impl<B: Backend> TensorCopy<B> {
source_array.buffer().borrow().deref(),
destination_array.buffer().borrow_mut().deref_mut(),
length as u32,
- command_buffer,
+ encoder,
);
Ok(())
}
diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
index 8a88a6741..f80eeadee 100644
--- a/crates/uzu/src/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -5,7 +5,7 @@ use std::{
use crate::{
array::{Array, ArrayContextExt},
- backends::common::{Backend, CommandBuffer, kernel::kv_cache_update::KVCacheUpdate},
+ backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
config::DecoderLayerType,
forward_pass::{
kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},
@@ -276,13 +276,13 @@ impl<B: Backend> CacheLayers<B> {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
kv_cache_update: &KVCacheUpdate<B>,
) {
let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);
for layer in self.data.iter_mut() {
if let Some(layer) = layer.as_transformer_mut() {
- layer.update_after_acceptance(accepted_suffix_indices, suffix_start, command_buffer, kv_cache_update);
+ layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);
} else if let Some(layer) = layer.as_short_conv() {
layer.commit_from_suffix_state_if_valid(short_conv_commit_index);
}
diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rs
index b60364064..6b0ffbe45 100644
--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs
@@ -1,7 +1,7 @@
use crate::{
array::{Array, ArrayCell, ArrayContextExt},
backends::common::{
- Backend, CommandBuffer,
+ Backend, Encoder,
kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},
},
utils::attention::fill_attention_bias,
@@ -163,7 +163,7 @@ impl<B: Backend> KVCacheLayer<B> {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
kv_cache_update: &KVCacheUpdate<B>,
) {
match &mut self.state {
@@ -182,7 +182,7 @@ impl<B: Backend> KVCacheLayer<B> {
let destination_indices: Vec<usize> =
(*prefix_len..*prefix_len + accepted_suffix_indices.len()).collect();
- self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);
+ self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);
},
KVCacheLayerState::Windowed {
@@ -204,7 +204,7 @@ impl<B: Backend> KVCacheLayer<B> {
destination_indices.push((*ring_length + *ring_offset + i) % *window_length);
}
- self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);
+ self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);
},
}
}
@@ -213,7 +213,7 @@ impl<B: Backend> KVCacheLayer<B> {
&self,
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
kv_cache_update: &KVCacheUpdate<B>,
) {
if source_indices == destination_indices {
@@ -230,7 +230,7 @@ impl<B: Backend> KVCacheLayer<B> {
value_shape: [v_shape[0], v_shape[1], v_shape[2]],
};
- let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, command_buffer);
+ let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, encoder);
}
pub fn register_accepted_tokens(
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index 8112db011..50e1c2ca2 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -3,7 +3,7 @@ use std::ops::{Deref, DerefMut};
use std::{cell::RefCell, collections::HashMap, rc::Rc};
#[cfg(feature = "tracing")]
-use crate::backends::common::{CommandBuffer, CommandBufferEncoding};
+use crate::backends::common::Encoder;
#[cfg(feature = "tracing")]
use crate::forward_pass::traces::ActivationTrace;
use crate::{
@@ -751,7 +751,7 @@ impl<B: Backend> ForwardPassState<B> {
#[cfg(feature = "tracing")]
pub fn encode_copy_array(
&self,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ encoder: &mut Encoder<B>,
source_array_id: ArrayId,
destination_array: RefCell<Array<B>>,
) {
@@ -765,6 +765,6 @@ impl<B: Backend> ForwardPassState<B> {
let copy_size_bytes = dst_borrow.size();
debug_assert_eq!(dst_borrow.size(), src_borrow.size());
- command_buffer.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);
+ encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);
}
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index ac08f8599..7245ecfda 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -17,8 +17,7 @@ use super::{
};
use crate::{
backends::common::{
- Backend, Buffer, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending, Context,
+ Backend, Buffer, CommandBuffer, Context, Encoder, Executable,
kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},
},
encodable_block::EncodingParameters,
@@ -67,7 +66,7 @@ pub struct LanguageModelGenerator<B: Backend> {
pub tokens: Vec<u64>,
pub context: LanguageModelGeneratorContext<B>,
- pre_encoded_task: Option<(TaskEncodingKey, <B::CommandBuffer as CommandBuffer>::Executable)>,
+ pre_encoded_task: Option<(TaskEncodingKey, Executable<B>)>,
registered_prefix_len: usize,
gpu_capture: GpuCaptureManager<B>,
}
@@ -509,28 +508,21 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
*sm = Some(sampling_method);
}
- let mut command_buffer = self
- .context
- .context
- .create_command_buffer()
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
- .start_encoding();
+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
// Wait on previous pass if this is a continuation
if is_continuation {
- command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
+ encoder.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
}
self.context
.executables
- .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)
+ .encode(&mut state, &EncodingParameters::new(), &mut encoder)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
// Encode sampling
- self.context
- .gpu_sampler
- .encode(&mut state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.gpu_sampler.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
// Copy sampled token: sampling_output → token_ids (for next pass)
// and sampling_output → results[slot] (for callback)
@@ -545,20 +537,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.token_copy_sampled.encode(
sampling_output_buf_borrow.deref(),
token_ids_buf_borrow.deref_mut(),
- &mut command_buffer,
+ &mut encoder,
);
let results_offset = slot * std::mem::size_of::<u32>();
self.context.token_copy_results.encode(
sampling_output_buf_borrow.deref(),
(results_buffer.borrow_mut().deref_mut(), results_offset),
- &mut command_buffer,
+ &mut encoder,
);
// Scatter + register for all transformer layers
self.context.cache_layers.borrow_mut().update_after_acceptance(
&[0],
None,
- &mut command_buffer,
+ &mut encoder,
&self.context.kv_cache_update,
);
self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
@@ -575,7 +567,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
mask_buf_borrow.deref_mut(),
update.unmask_col,
update.mask_col,
- &mut command_buffer,
+ &mut encoder,
);
}
}
@@ -584,7 +576,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// Signal event for next pass
let next_counter = current_counter + 1;
- command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);
+ encoder.encode_signal_event(&self.context.async_buffers.event, next_counter);
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
@@ -596,10 +588,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
on_complete(token);
};
- command_buffer.add_completion_handler(handler);
+ encoder.add_completion_handler(handler);
drop(token_ids_binding);
- let pending = command_buffer.end_encoding().submit();
+ let pending = encoder.end_encoding().submit();
if should_capture {
pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
@@ -825,27 +817,20 @@ impl<B: Backend> LanguageModelGenerator<B> {
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
sample: bool,
- ) -> Result<<B::CommandBuffer as CommandBuffer>::Executable, Error> {
- let mut command_buffer = self
- .context
- .context
- .create_command_buffer()
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
- .start_encoding();
+ ) -> Result<Executable<B>, Error> {
+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
self.context
.executables
- .encode(state, parameters, &mut command_buffer)
+ .encode(state, parameters, &mut encoder)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
if sample {
- self.context
- .gpu_sampler
- .encode(state, &mut command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ self.context.gpu_sampler.encode(state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
- let executable = command_buffer.end_encoding();
+ let executable = encoder.end_encoding();
Ok(executable)
}
@@ -876,20 +861,20 @@ impl<B: Backend> LanguageModelGenerator<B> {
suffix_start: Option<usize>,
wait_until_completed: bool,
) -> Result<(), Error> {
- let mut command_buffer =
- self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
{
let mut cache_layers = self.context.cache_layers.borrow_mut();
cache_layers.update_after_acceptance(
accepted_token_indices,
suffix_start,
- &mut command_buffer,
+ &mut encoder,
&self.context.kv_cache_update,
);
}
- let pending = command_buffer.end_encoding().submit();
+ let pending = encoder.end_encoding().submit();
if wait_until_completed {
pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 848bcb579..62d18985d 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -18,10 +18,7 @@ use num_traits::NumCast;
use crate::{
ArrayElement, DataType,
array::Array,
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- kernel::kv_cache_update::KVCacheUpdate,
- },
+ backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},
classifier::Classifier,
config::ModelMetadata,
encodable_block::{EncodingParameters, Sampling},
@@ -273,13 +270,13 @@ impl<B: Backend> TraceValidator<B> {
None,
);
- let mut command_buffer =
- ctx.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder =
+ Encoder::<B>::new(ctx.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
ctx.executables
- .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)
+ .encode(&mut state, &EncodingParameters::new(), &mut encoder)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- let pending = command_buffer.end_encoding().submit();
+ let pending = encoder.end_encoding().submit();
pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
let traces = state.traces().clone();
diff --git a/crates/uzu/tests/integration/util/mod.rs b/crates/uzu/tests/integration/util/mod.rs
index 6b0e55f15..b67b341c8 100644
--- a/crates/uzu/tests/integration/util/mod.rs
+++ b/crates/uzu/tests/integration/util/mod.rs
@@ -1,2 +1,3 @@
+#[cfg_attr(not(feature = "tracing"), allow(dead_code))]
pub mod path;
pub mod speculator;
diff --git a/crates/uzu/tests/performance/matmul/bench.rs b/crates/uzu/tests/performance/matmul/bench.rs
index 224a9aeb2..b6489d6d0 100644
--- a/crates/uzu/tests/performance/matmul/bench.rs
+++ b/crates/uzu/tests/performance/matmul/bench.rs
@@ -4,8 +4,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending, Context,
+ Backend, Encoder,
kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
metal::Metal,
@@ -39,7 +38,7 @@ fn encode_and_run(
b_buffer: &Buf,
d_buffer: &mut Buf,
) -> Result<f64, BenchError> {
- let mut command_buffer = context.create_command_buffer().map_err(|_| BenchError::CommandBuffer)?.start_encoding();
+ let mut encoder = Encoder::new(context).map_err(|_| BenchError::CommandBuffer)?;
kernel.encode(
context,
@@ -57,11 +56,10 @@ fn encode_and_run(
leading_dimension_d: shape.output_dim as i32,
transpose_b: true,
},
- &mut command_buffer,
+ &mut encoder,
);
- let completed =
- command_buffer.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;
+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;
completed.gpu_execution_time().map(|duration| duration.as_secs_f64() * 1000.0).ok_or(BenchError::GpuTimestamps)
}
diff --git a/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
index 9afec7e7d..4fbaab822 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
@@ -8,7 +8,7 @@ use ndarray::{Array, Array3, s};
use super::*;
use crate::backends::{
- common::{CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context},
+ common::{Context, Encoder},
metal::Metal,
};
@@ -104,8 +104,8 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
value_shape: [num_heads, seq_len, head_dim],
};
- let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();
- match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut command_buffer) {
+ let mut encoder = Encoder::new(context).unwrap();
+ match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut encoder) {
Ok(_) => {},
Err(e) => {
println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);
@@ -113,7 +113,7 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
},
}
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;
let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;
diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
index e60458411..38e8c653f 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
@@ -11,8 +11,7 @@ use crate::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context,
+ Backend, Context, Encoder,
gpu_types::QuantizationMode,
kernel::quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
@@ -514,9 +513,9 @@ fn execute_quantized_matmul(
output_dim,
quantization_type,
};
- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
- kernel.encode(&mut command_buffer, args).unwrap();
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(ctx).unwrap();
+ kernel.encode(&mut encoder, args).unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
}
@@ -534,9 +533,9 @@ fn execute_quantized_matmul(
output_dim,
quantization_type,
};
- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
- kernel.encode(&mut command_buffer, args).unwrap();
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(ctx).unwrap();
+ kernel.encode(&mut encoder, args).unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
let elapsed = start.elapsed();
diff --git a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
index 83cf9fa95..a9955faed 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
@@ -10,8 +10,7 @@ use crate::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels,
+ Backend, Context, Encoder, Kernels,
gpu_types::ActivationType,
kernel::{
Conv1dScanKernel,
@@ -248,9 +247,9 @@ fn run_prefill_kernel_mode(
head_dim: fixture.head_dim,
};
- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
- kernel.encode(&mut command_buffer, args, mode);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(ctx).unwrap();
+ kernel.encode(&mut encoder, args, mode);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let y_vec = read_buffer(&y_buf, fixture.total_x);
let state_vec = read_buffer(&state_buf, fixture.total_state);
@@ -318,7 +317,7 @@ fn run_conv_scan_once(
write_buffer(&padded_buf, &host);
}
- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
+ let mut encoder = Encoder::new(ctx).unwrap();
let mut b_out = y_buf.clone();
let mut c_out = y_buf.clone();
let mut state_out = scratch_buf.as_ref().unwrap_or(&state_buf).clone();
@@ -338,17 +337,17 @@ fn run_conv_scan_once(
channels as u32,
0u32,
ActivationType::SILU,
- &mut command_buffer,
+ &mut encoder,
);
if let Some(ref scratch) = scratch_buf {
let bytes = channels * tap_count * size_of::<f32>();
if bytes > 0 {
- command_buffer.encode_copy(scratch, &mut state_buf, bytes);
+ encoder.encode_copy(scratch, &mut state_buf, bytes);
}
}
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let y_vec = read_buffer(&y_buf, total_x);
let state_vec = read_buffer(&state_buf, total_state);
diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rs
index 86c4c5c66..9f070d94e 100644
--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs
@@ -12,8 +12,7 @@ use crate::{
DataType,
backends::{
common::{
- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending, Context, Kernels,
+ Backend, Context, Encoder, Kernels,
kernel::{
AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,
attention::{AttentionGemmArguments, AttentionGemmBlock},
@@ -317,7 +316,7 @@ fn run_single_pass_attention(
.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -347,9 +346,9 @@ fn run_single_pass_attention(
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -386,7 +385,7 @@ fn run_single_pass_attention_with_is_causal(
.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -415,9 +414,9 @@ fn run_single_pass_attention_with_is_causal(
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -453,7 +452,7 @@ fn run_gemm_attention(
.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");
let args = AttentionGemmArguments {
queries_buffer: &query_buffer,
@@ -473,8 +472,8 @@ fn run_gemm_attention(
scale,
};
- kernel.encode(context, &mut command_buffer, args)?;
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ kernel.encode(context, &mut encoder, args)?;
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -923,7 +922,7 @@ fn run_two_pass_attention(
)
.expect("Failed to create buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -954,7 +953,7 @@ fn run_two_pass_attention(
mask_q_seq_stride,
mask_head_stride,
sinks_buffer.as_ref().map(|b| b),
- &mut command_buffer,
+ &mut encoder,
);
kernel_pass2.encode(
&partials_buffer,
@@ -963,9 +962,9 @@ fn run_two_pass_attention(
&mut output_buffer,
num_heads as u32,
seq_len as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };
@@ -1158,7 +1157,7 @@ fn perf_two_pass_attention() {
.expect("Failed to create buffer");
// ---- Launch and time ----
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
@@ -1183,7 +1182,7 @@ fn perf_two_pass_attention() {
None,
None,
sinks_buffer.as_ref().map(|b| b),
- &mut command_buffer,
+ &mut encoder,
);
kernel_pass2.encode(
&partials_buffer,
@@ -1192,11 +1191,11 @@ fn perf_two_pass_attention() {
&mut output_buffer,
num_heads as u32,
suffix_length as u32,
- &mut command_buffer,
+ &mut encoder,
);
// Time both host-side and GPU execution
let host_timer = Instant::now();
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
match completed.gpu_execution_time() {
diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
index 70ad296bf..329d2fbb3 100644
--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
@@ -9,8 +9,7 @@ use crate::{
DataType,
backends::{
common::{
- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending, Context, Kernels,
+ Backend, Context, Encoder, Kernels,
kernel::{
MinPKernel, TemperatureKernel,
sampling::{ArgmaxStrategy, SamplingKernel},
@@ -76,7 +75,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
.new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
&mut logits_buffer,
@@ -88,10 +87,10 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
SamplingMethod::Greedy,
batch_size,
vocab_size,
- &mut command_buffer,
+ &mut encoder,
)
.expect("Argmax sampling should succeed");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Check results
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
@@ -166,7 +165,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
.expect("Failed to create buffer");
// ---- Launch once and time ----
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
@@ -179,13 +178,13 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
SamplingMethod::Greedy,
BATCH,
VOCAB,
- &mut command_buffer,
+ &mut encoder,
)
.expect("encode");
// Time both host-side and GPU execution
let host_timer = Instant::now();
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
match completed.gpu_execution_time() {
@@ -282,8 +281,7 @@ fn test_categorical_sampling() {
.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
@@ -301,10 +299,10 @@ fn test_categorical_sampling() {
},
batch_size,
vocab_size,
- &mut command_buffer,
+ &mut encoder,
)
.expect("Categorical sampling should succeed");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };
@@ -419,8 +417,7 @@ fn test_categorical_sampling_statistical() {
.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
@@ -438,10 +435,10 @@ fn test_categorical_sampling_statistical() {
},
BATCH,
VOCAB,
- &mut command_buffer,
+ &mut encoder,
)
.expect("encode");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let result_ptr = output_buffer.contents().as_ptr() as *const u32;
let results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH) };
@@ -521,7 +518,7 @@ fn perf_categorical_128k_vocab() {
.expect("Failed to create buffer");
// Launch and time
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
@@ -539,12 +536,12 @@ fn perf_categorical_128k_vocab() {
},
BATCH,
VOCAB,
- &mut command_buffer,
+ &mut encoder,
)
.expect("encode");
let host_timer = Instant::now();
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
match completed.gpu_execution_time() {
@@ -606,16 +603,9 @@ fn test_temperature_gpu_cpu_match() {
.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(
- Some(&logits_buffer),
- &mut processed_buffer,
- BATCH as u32,
- VOCAB as u32,
- TEMPERATURE,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;
let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };
@@ -673,9 +663,9 @@ fn test_minp_gpu_cpu_match() {
.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut command_buffer);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let results_ptr = processed_buffer.contents().as_ptr() as *const f32;
let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };
@@ -756,8 +746,7 @@ fn test_minp_sampling_exact_match(
.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
&mut logits_buf,
@@ -774,10 +763,10 @@ fn test_minp_sampling_exact_match(
},
batch_size,
vocab_size,
- &mut command_buffer,
+ &mut encoder,
)
.expect("encode");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let ptr = output_buf.contents().as_ptr() as *const u32;
let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };
diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
index 01d5baa5f..074ca8fb6 100644
--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
@@ -4,10 +4,7 @@ use crate::{
DataType,
array::ArrayContextExt,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, kernel::kv_cache_update::KVCacheUpdate,
- },
+ common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
metal::Metal,
},
forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},
@@ -186,15 +183,15 @@ fn run_scenario(
},
};
- let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();
+ let mut encoder = Encoder::new(context).unwrap();
layer.update_after_acceptance(
&scenario.accepted_suffix_indices,
scenario.suffix_start,
- &mut command_buffer,
+ &mut encoder,
&kv_cache_update,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
layer.register_accepted_tokens(&scenario.accepted_token_positions);
diff --git a/crates/uzu/tests/unit/kernel/activation_mul_test.rs b/crates/uzu/tests/unit/kernel/activation_mul_test.rs
index 5f4cba53b..f55d292e5 100644
--- a/crates/uzu/tests/unit/kernel/activation_mul_test.rs
+++ b/crates/uzu/tests/unit/kernel/activation_mul_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel},
cpu::Cpu,
},
};
@@ -58,16 +55,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let fused_up_array = context.create_array_from(&[fused_len], &input.fused_up, "");
let hidden_array = context.create_array_uninitialized(&[out_len], T::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
fused_up_array.buffer().borrow().deref(),
hidden_array.buffer().borrow_mut().deref_mut(),
input.h,
input.m,
input.act_type,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
hidden_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs
index 7a9725809..e3216cd1b 100644
--- a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs
+++ b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, kernel::attention::AttentionGemmArguments,
- },
+ common::{Backend, Context, Encoder, kernel::attention::AttentionGemmArguments},
cpu::Cpu,
},
};
@@ -148,10 +145,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
scale: input.scale,
};
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- block.encode(&context, &mut command_buffer, args).expect("Failed to encode AttentionGemm");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ block.encode(&context, &mut encoder, args).expect("Failed to encode AttentionGemm");
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
output_array.as_slice().to_vec()
diff --git a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
index b59aa67a0..8fc02f929 100644
--- a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
+++ b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::AttentionSinglePassKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},
cpu::Cpu,
},
};
@@ -133,7 +130,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);
let mask_head_stride: Option<u32> = input.has_mask.then_some(0);
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
queries_array.buffer().borrow().deref(),
keys_array.buffer().borrow().deref(),
@@ -153,9 +150,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
None::<&B::Buffer>,
input.num_heads,
input.suffix_length,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs
index f06cbfda9..e16d1d23d 100644
--- a/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs
+++ b/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::AttentionUpdateKVCacheKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::AttentionUpdateKVCacheKernel},
cpu::Cpu,
},
};
@@ -53,7 +50,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let key_cache_array = context.create_array_from(&[cache_size], &input.key_cache, "");
let value_cache_array = context.create_array_from(&[cache_size], &input.value_cache, "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
rotated_keys_buffer,
qkv_array.buffer().borrow().deref(),
@@ -65,9 +62,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
input.suffix_length,
input.prefix_segment_length,
input.max_sequence_length,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
(key_cache_array.as_slice().to_vec(), value_cache_array.as_slice().to_vec())
}
diff --git a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
index cde59daf1..25f3f9d6d 100644
--- a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
+++ b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs
@@ -5,10 +5,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::SigmoidGateKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::SigmoidGateKernel},
cpu::Cpu,
},
};
@@ -32,15 +29,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice(), "gate");
let output_array = context.create_array_from(&[size], &output_data.to_vec().into_boxed_slice(), "output");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
let total_elements = config.suffix_length * config.num_heads * config.head_dim;
kernel.encode(
&*gate_array.buffer().borrow(),
output_array.buffer().borrow_mut().deref_mut(),
total_elements,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/audio/audio_test.rs b/crates/uzu/tests/unit/kernel/audio/audio_test.rs
index da86155f2..48aff9f65 100644
--- a/crates/uzu/tests/unit/kernel/audio/audio_test.rs
+++ b/crates/uzu/tests/unit/kernel/audio/audio_test.rs
@@ -9,8 +9,7 @@ use uzu::{
ArrayContextExt, DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels,
+ Backend, Context, Encoder, Kernels,
kernel::{
AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,
AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,
@@ -102,7 +101,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {
}
}
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input.buffer().borrow().deref(),
weight.buffer().borrow().deref(),
@@ -119,9 +118,9 @@ fn audio_conv1d_replicate_matches_reference_f32() {
padding as i32,
1_i32,
batch_size as i32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let got = output.as_slice::<f32>();
for index in 0..output_len {
@@ -203,7 +202,7 @@ fn audio_causal_conv1d_matches_reference_f32() {
}
}
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input.buffer().borrow().deref(),
weight.buffer().borrow().deref(),
@@ -216,9 +215,9 @@ fn audio_causal_conv1d_matches_reference_f32() {
kernel_size as i32,
dilation as i32,
batch_size as i32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let got = output.as_slice::<f32>();
for index in 0..output_len {
@@ -313,7 +312,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
}
}
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input.buffer().borrow().deref(),
weight.buffer().borrow().deref(),
@@ -327,9 +326,9 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
stride as i32,
groups as i32,
batch_size as i32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let got = output.as_slice::<f32>();
for index in 0..output_len {
@@ -358,15 +357,15 @@ fn audio_leaky_relu_matches_reference_f32() {
let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input.buffer().borrow().deref(),
output.buffer().borrow_mut().deref_mut(),
n as i32,
slope,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let got = output.as_slice::<f32>();
for index in 0..n {
@@ -394,14 +393,9 @@ fn audio_tanh_matches_reference_f32() {
let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
- kernel.encode(
- input.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- n as i32,
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(input.buffer().borrow().deref(), output.buffer().borrow_mut().deref_mut(), n as i32, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let got = output.as_slice::<f32>();
for index in 0..n {
@@ -433,22 +427,22 @@ fn audio_add_and_scale_match_reference_f32() {
let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");
let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
add_kernel.encode(
a.buffer().borrow().deref(),
b.buffer().borrow().deref(),
sum.buffer().borrow_mut().deref_mut(),
n as i32,
- &mut command_buffer,
+ &mut encoder,
);
scale_kernel.encode(
sum.buffer().borrow().deref(),
scaled.buffer().borrow_mut().deref_mut(),
n as i32,
scale_value,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let sum_got = sum.as_slice::<f32>();
let scaled_got = scaled.as_slice::<f32>();
@@ -489,16 +483,16 @@ fn audio_clamp_matches_reference_f32() {
let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input.buffer().borrow().deref(),
output.buffer().borrow_mut().deref_mut(),
n as i32,
min_value,
max_value,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let got = output.as_slice::<f32>();
for index in 0..n {
@@ -534,7 +528,7 @@ fn audio_half_snake_matches_reference_f32() {
let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input.buffer().borrow().deref(),
alpha.buffer().borrow().deref(),
@@ -545,9 +539,9 @@ fn audio_half_snake_matches_reference_f32() {
negative_slope,
eps,
batch_size as i32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let got = output.as_slice::<f32>();
for batch in 0..batch_size {
@@ -600,7 +594,7 @@ fn audio_fsq_decode_matches_reference() {
let output =
context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
tokens.buffer().borrow().deref(),
output.buffer().borrow_mut().deref_mut(),
@@ -610,9 +604,9 @@ fn audio_fsq_decode_matches_reference() {
codebook_dim as i32,
&num_levels,
batch_size as i32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let expected =
reference_fsq_decode(&[0, 7, 11, 3, 5, 9], 2, batch_size, num_groups, seq_len, codebook_dim, &num_levels);
@@ -658,7 +652,7 @@ fn audio_fsq_encode_matches_reference() {
let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");
- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input.buffer().borrow().deref(),
tokens.buffer().borrow_mut().deref_mut(),
@@ -670,9 +664,9 @@ fn audio_fsq_encode_matches_reference() {
&dim_base_index,
eps,
batch_size as i32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let expected = reference_fsq_encode(
&input_values,
diff --git a/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs b/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs
index cbb24e518..d5e279fa1 100644
--- a/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs
+++ b/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs
@@ -7,10 +7,7 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement,
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- Kernels, kernel::FullPrecisionEmbeddingLookupKernel,
- },
+ backends::common::{Backend, Context, Encoder, Kernels, kernel::FullPrecisionEmbeddingLookupKernel},
};
struct Input<T: ArrayElement + Float> {
@@ -62,7 +59,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights, "");
let output_array = context.create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to get command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");
kernel.encode(
token_ids_array.buffer().borrow().deref(),
weights_array.buffer().borrow().deref(),
@@ -71,9 +68,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
input.vocab_size as u32,
input.model_dim as u32,
input.input_scale,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs
index 444f0b17c..4fdfc6002 100644
--- a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs
@@ -9,8 +9,7 @@ use uzu::{
ArrayContextExt, ArrayElement,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context,
+ Backend, Context, Encoder,
kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
cpu::Cpu,
@@ -68,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())
.expect("Failed to create MatmulKernel");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
&context,
MatmulArguments {
@@ -85,9 +84,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
leading_dimension_d: n,
transpose_b: true,
},
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(d_ref);
d_array.as_slice().to_vec()
diff --git a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
index c868918d6..a0772b176 100644
--- a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs
@@ -10,8 +10,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context,
+ Backend, Context, Encoder,
kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
metal::Metal,
@@ -49,7 +48,7 @@ fn run_metal_matmul(
let mut kernel =
<<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");
- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
+ let mut encoder = Encoder::new(ctx).unwrap();
kernel.encode(
ctx,
MatmulArguments {
@@ -66,9 +65,9 @@ fn run_metal_matmul(
leading_dimension_d: n as i32,
transpose_b,
},
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
unsafe {
let ptr = d_buf.contents().as_ptr() as *const bf16;
diff --git a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
index 103c5d53a..e2a476fdf 100644
--- a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs
@@ -9,8 +9,7 @@ use uzu::{
ArrayContextExt, ArrayElement,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context,
+ Backend, Context, Encoder,
kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
},
cpu::Cpu,
@@ -68,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type())
.expect("Failed to create MatmulKernel");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
&context,
MatmulArguments {
@@ -85,9 +84,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
leading_dimension_d: n,
transpose_b: true,
},
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
drop(d_ref);
d_array.as_slice().to_vec()
diff --git a/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rs
index ce4c5badd..acc5932cb 100644
--- a/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rs
@@ -8,8 +8,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels,
+ Backend, Encoder, Kernels,
kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},
},
metal::Metal,
@@ -76,7 +75,7 @@ fn gen_topk_ids_from_logits(
let mut topk_probs_buf = alloc_buffer::<bf16>(ctx, t * k);
// Use fused router+topk kernel
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16)
.expect("router_topk kernel");
router_topk.encode(
@@ -90,9 +89,9 @@ fn gen_topk_ids_from_logits(
e as u32,
k as u32,
true,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let ids_ptr = topk_ids_buf.contents().as_ptr() as *const i32;
let ids = unsafe { std::slice::from_raw_parts(ids_ptr, t * k) };
@@ -124,8 +123,7 @@ fn test_counts_offsets_fused_parity_random() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let mut command_buffer =
- ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
kernel.encode(
&topk_ids_buf,
@@ -135,10 +133,10 @@ fn test_counts_offsets_fused_parity_random() {
t as u32,
e as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Verify offsets
let offsets_ptr = offsets_buf.contents().as_ptr() as *const u32;
@@ -174,7 +172,7 @@ fn test_counts_offsets_fused_edge_cases() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -183,9 +181,9 @@ fn test_counts_offsets_fused_edge_cases() {
t as u32,
e as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };
@@ -204,7 +202,7 @@ fn test_counts_offsets_fused_edge_cases() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -213,9 +211,9 @@ fn test_counts_offsets_fused_edge_cases() {
t as u32,
e as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };
assert!(offsets_gpu.iter().all(|&v| v == 0));
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rs
index 8b038f7f0..e3bceb55d 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rs
@@ -7,8 +7,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels,
+ Backend, Encoder, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
@@ -349,7 +348,7 @@ fn run_moe_parity_test_internal(
// Encode ALL kernels in one command buffer
eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
// Router + TopK (fused kernel)
let router_topk =
@@ -365,7 +364,7 @@ fn run_moe_parity_test_internal(
e as u32,
k as u32,
true,
- &mut command_buffer,
+ &mut encoder,
);
let fused_kernel =
@@ -378,7 +377,7 @@ fn run_moe_parity_test_internal(
t as u32,
e as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)
@@ -391,7 +390,7 @@ fn run_moe_parity_test_internal(
num_blocks as u32,
num_tiles as u32,
0u32,
- &mut command_buffer,
+ &mut encoder,
);
let scatter_map_kernel =
@@ -411,12 +410,12 @@ fn run_moe_parity_test_internal(
num_blocks as u32,
num_tiles as u32,
&mut tok2row_buf,
- &mut command_buffer,
+ &mut encoder,
);
let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");
gather.encode(
- &mut command_buffer,
+ &mut encoder,
DataType::BF16,
MoeGatherArguments {
x_buffer: &x_buf,
@@ -464,7 +463,7 @@ fn run_moe_parity_test_internal(
silu_alpha,
data_type: DataType::BF16,
};
- experts.encode(&mut command_buffer, args);
+ experts.encode(&mut encoder, args);
let finalize =
<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
@@ -476,11 +475,11 @@ fn run_moe_parity_test_internal(
t as u32,
d_model as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
eprintln!("[E2E] GPU execution completed");
// Read GPU output
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rs
index b7c5377da..844a5d72b 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rs
@@ -8,8 +8,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context,
+ Backend, Encoder,
kernel::moe::{
MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,
@@ -108,9 +107,9 @@ fn run_decode_case(
let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);
for _ in 0..warmup {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
experts_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -140,15 +139,15 @@ fn run_decode_case(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
experts_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -178,7 +177,7 @@ fn run_decode_case(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -270,7 +269,7 @@ fn run_two_pass_prefill_case(
let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);
for _ in 0..warmup {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -299,14 +298,14 @@ fn run_two_pass_prefill_case(
silu_alpha: 1.702,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut command_buffer, args);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ experts_kernel.encode(&mut encoder, args);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -335,8 +334,8 @@ fn run_two_pass_prefill_case(
silu_alpha: 1.702,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut command_buffer, args);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ experts_kernel.encode(&mut encoder, args);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -411,9 +410,9 @@ fn run_fused_single_token_case(
let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);
for _ in 0..warmup {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
fused_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -436,15 +435,15 @@ fn run_fused_single_token_case(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
fused_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -467,7 +466,7 @@ fn run_fused_single_token_case(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -589,9 +588,9 @@ fn run_indirect_decode_timed(
let mut row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);
for _ in 0..warmup {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
experts_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -621,15 +620,15 @@ fn run_indirect_decode_timed(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
experts_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -659,7 +658,7 @@ fn run_indirect_decode_timed(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
@@ -706,9 +705,9 @@ fn run_fused_decode_timed(
let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);
for _ in 0..warmup {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
fused_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -731,15 +730,15 @@ fn run_fused_decode_timed(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
}
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
fused_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -762,7 +761,7 @@ fn run_fused_decode_timed(
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
times.push(start.elapsed().as_secs_f64() * 1000.0);
}
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rs
index 701784cb9..2a3117213 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rs
@@ -15,7 +15,7 @@ use uzu::{
DataType,
backends::{
common::{
- CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Encoder,
kernel::moe::{
MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,
MoeExpertsTwoPassPrefillBlock,
@@ -419,13 +419,13 @@ fn test_two_pass_decode_correctness() {
// Execute 2-pass decode kernel
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
const K_TILE: usize = 64;
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;
experts_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -456,7 +456,7 @@ fn test_two_pass_decode_correctness() {
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Read GPU partial output and do CPU finalize (weighted sum)
let y_partial_gpu =
@@ -562,9 +562,9 @@ fn test_two_pass_decode_multi_token() {
let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
experts_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -594,7 +594,7 @@ fn test_two_pass_decode_multi_token() {
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let y_partial_gpu =
unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
@@ -663,7 +663,7 @@ fn test_two_pass_prefill_correctness() {
let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
let args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -692,8 +692,8 @@ fn test_two_pass_prefill_correctness() {
silu_alpha,
data_type: DataType::BF16,
};
- experts_kernel.encode(&mut command_buffer, args);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ experts_kernel.encode(&mut encoder, args);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let y_partial_gpu =
unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };
@@ -791,10 +791,10 @@ fn test_fused_single_token_decode() {
// Run fused decode kernel
let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
fused_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -818,7 +818,7 @@ fn test_fused_single_token_decode() {
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Read GPU output
let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };
@@ -914,9 +914,9 @@ fn test_fused_single_token_k4() {
let mut y_buf = alloc_buffer::<bf16>(&ctx, d_model);
let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
fused_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -939,7 +939,7 @@ fn test_fused_single_token_k4() {
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs
index 583b5e376..5f9c1c975 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs
@@ -6,10 +6,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::MoeFinalizeKernel,
- },
+ common::{Backend, Encoder, Kernels, kernel::MoeFinalizeKernel},
metal::Metal,
},
};
@@ -93,7 +90,7 @@ fn test_finalize_correctness() {
// Execute finalize kernel
let finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16)
.expect("finalize kernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
finalize.encode(
&tok2row_buf,
&probs_buf,
@@ -102,9 +99,9 @@ fn test_finalize_correctness() {
t as u32,
d_model as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let y_gpu = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs
index b38c936ef..5e4bf1399 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs
@@ -7,7 +7,7 @@ use uzu::{
DataType,
backends::{
common::{
- CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Encoder,
kernel::moe::{MoeGatherArguments, MoeGatherKernels},
},
metal::Metal,
@@ -81,9 +81,9 @@ fn test_gather_correctness() {
// Execute gather kernel using kernel struct
let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
gather.encode(
- &mut command_buffer,
+ &mut encoder,
DataType::BF16,
MoeGatherArguments {
x_buffer: &x_buf,
@@ -95,7 +95,7 @@ fn test_gather_correctness() {
d_model,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let x_gpu =
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs
index e87707aba..01a5f181f 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs
@@ -8,8 +8,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels,
+ Backend, Encoder, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
@@ -123,8 +122,7 @@ fn test_moe_e2e_decode_perf() {
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
- let mut command_buffer =
- ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -136,9 +134,9 @@ fn test_moe_e2e_decode_perf() {
e as u32,
k as u32,
true,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
fused_perf.print();
@@ -182,8 +180,7 @@ fn test_moe_e2e_prefill_perf() {
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
- let mut command_buffer =
- ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -195,9 +192,9 @@ fn test_moe_e2e_prefill_perf() {
e as u32,
k as u32,
true,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
fused_perf.print();
@@ -309,7 +306,7 @@ fn test_moe_pipeline_breakdown_decode() {
// Testing: Router + TopK + Counts+Offsets (FUSED)
let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
router_topk_fused_kernel.encode(
&x_buf,
&router_w_buf,
@@ -321,13 +318,13 @@ fn test_moe_pipeline_breakdown_decode() {
e as u32,
k as u32,
true,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
counts_offsets_kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -336,13 +333,13 @@ fn test_moe_pipeline_breakdown_decode() {
t as u32,
e as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
let scatter_perf = time_kernel("Scatter", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
scatter_bases_kernel.encode(
&partials_buf,
&mut block_bases_buf,
@@ -351,7 +348,7 @@ fn test_moe_pipeline_breakdown_decode() {
num_blocks as u32,
num_tiles as u32,
0u32,
- &mut command_buffer,
+ &mut encoder,
);
scatter_map_kernel.encode(
&topk_ids_buf,
@@ -367,15 +364,15 @@ fn test_moe_pipeline_breakdown_decode() {
num_blocks as u32,
num_tiles as u32,
&mut tok2row_buf,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
let gather_perf = time_kernel("Gather", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
gather_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
DataType::BF16,
MoeGatherArguments {
x_buffer: &x_buf,
@@ -387,13 +384,13 @@ fn test_moe_pipeline_breakdown_decode() {
d_model,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
experts_kernel.encode(
- &mut command_buffer,
+ &mut encoder,
MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
@@ -423,11 +420,11 @@ fn test_moe_pipeline_breakdown_decode() {
data_type: DataType::BF16,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
let finalize_perf = time_kernel("Finalize", 2, 5, || {
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
finalize_kernel.encode(
&tok2row_buf,
&topk_probs_buf,
@@ -436,9 +433,9 @@ fn test_moe_pipeline_breakdown_decode() {
t as u32,
d_model as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
});
// Print results
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs
index 75100831f..2f73636a0 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs
@@ -6,10 +6,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::MoeRouterTopKKernel,
- },
+ common::{Backend, Encoder, Kernels, kernel::MoeRouterTopKKernel},
metal::Metal,
},
};
@@ -160,7 +157,7 @@ fn run_router_topk_once(
// For BFloat16 kernel, probs buffer must be bf16, not f32
let mut probs_buf = alloc_buffer::<bf16>(ctx, t * k);
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");
kernel.encode(
&input_buf,
&weight_buf,
@@ -172,9 +169,9 @@ fn run_router_topk_once(
e as u32,
k as u32,
renorm,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let ids_ptr = ids_buf.contents().as_ptr() as *const i32;
let probs_ptr = probs_buf.contents().as_ptr() as *const bf16;
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs
index e532d764a..febdb4e2b 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs
@@ -7,8 +7,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels,
+ Backend, Encoder, Kernels,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,
MoeScatterBucketsKernel,
@@ -85,7 +84,7 @@ fn test_scatter_buckets_parity() {
// Use fused router+topk kernel
let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
.expect("router_topk");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
router_topk.encode(
&input_buf,
&weight_buf,
@@ -97,9 +96,9 @@ fn test_scatter_buckets_parity() {
e as u32,
k as u32,
true,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Read back CPU probs for reference compare
let probs_bf16 =
@@ -114,7 +113,7 @@ fn test_scatter_buckets_parity() {
let fused_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
fused_kernel.encode(
&topk_ids_buf,
&mut offsets_buf,
@@ -123,9 +122,9 @@ fn test_scatter_buckets_parity() {
t as u32,
e as u32,
k as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Partials already created by fused kernel above
let num_blocks = 1; // Fused kernel uses single block
@@ -138,7 +137,7 @@ fn test_scatter_buckets_parity() {
let scatter_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, DataType::BF16)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
let mut block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);
scatter_bases_kernel.encode(
@@ -149,16 +148,16 @@ fn test_scatter_buckets_parity() {
num_blocks as u32,
num_tiles as u32,
0u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let sumk = unsafe { std::slice::from_raw_parts(sumk_buf.contents().as_ptr() as *const u32, 1) }[0] as usize;
let mut out_ids_buf = alloc_buffer::<i32>(&ctx, sumk);
let mut out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
scatter_kernel.encode(
&topk_ids_buf,
&topk_probs_buf,
@@ -172,10 +171,10 @@ fn test_scatter_buckets_parity() {
k as u32,
num_blocks as u32,
num_tiles as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let out_ids = unsafe { std::slice::from_raw_parts(out_ids_buf.contents().as_ptr() as *const i32, sumk) };
let out_probs_h = unsafe { std::slice::from_raw_parts(out_probs_buf.contents().as_ptr() as *const bf16, sumk) };
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rs
index ab7eac9f1..57c6e6e19 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rs
@@ -4,7 +4,7 @@ use metal::MTLBuffer;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::backends::{
common::{
- CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ Encoder,
kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},
},
metal::Metal,
@@ -38,16 +38,16 @@ fn test_tile_counts_correctness() {
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
tile_kernel.encode_counts(
- &mut command_buffer,
+ &mut encoder,
MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &mut tile_counts_buf,
e,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let tile_counts_gpu =
@@ -82,9 +82,9 @@ fn test_tile_scan_correctness() {
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
tile_kernel.encode_scan(
- &mut command_buffer,
+ &mut encoder,
MoeTileScanArguments {
tile_counts_buffer: &tile_counts_buf,
tile_offsets_buffer: &mut tile_offsets_buf,
@@ -92,7 +92,7 @@ fn test_tile_scan_correctness() {
e,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
// Compare
let tile_offsets_gpu =
@@ -121,16 +121,16 @@ fn test_tile_edge_cases() {
let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
tile_kernel.encode_counts(
- &mut command_buffer,
+ &mut encoder,
MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &mut tile_counts_buf,
e,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let tile_counts_gpu =
unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };
@@ -151,16 +151,16 @@ fn test_tile_edge_cases() {
let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
tile_kernel.encode_counts(
- &mut command_buffer,
+ &mut encoder,
MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &mut tile_counts_buf,
e,
},
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let tile_counts_gpu =
unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };
diff --git a/crates/uzu/tests/unit/kernel/qk_norm_test.rs b/crates/uzu/tests/unit/kernel/qk_norm_test.rs
index ad6d5b038..ebf13d876 100644
--- a/crates/uzu/tests/unit/kernel/qk_norm_test.rs
+++ b/crates/uzu/tests/unit/kernel/qk_norm_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::QKNormKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::QKNormKernel},
cpu::Cpu,
},
};
@@ -132,7 +129,7 @@ fn get_output<
let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");
let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
None::<&B::Buffer>,
scales_array.buffer().borrow().deref(),
@@ -146,10 +143,9 @@ fn get_output<
input.head_offset,
input.head_count,
input.full_layer,
- &mut command_buffer,
+ &mut encoder,
);
-
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
qkv_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
index db9891942..75d5c0158 100644
--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
@@ -10,10 +10,7 @@ use rand::{RngExt, SeedableRng, prelude::StdRng};
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending, Context, Kernels, kernel::RMSNormKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::RMSNormKernel},
cpu::Cpu,
},
};
@@ -144,7 +141,7 @@ fn get_output<
false => context.create_array_uninitialized(&[input_size], OutputT::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input_buffer,
scales_array.buffer().borrow().deref(),
@@ -154,12 +151,11 @@ fn get_output<
input.epsilon,
input.scale_offset,
input.full_layer,
- &mut command_buffer,
+ &mut encoder,
);
let instant = Instant::now();
- let completed =
- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ let completed = encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
let host_elapsed_ms = instant.elapsed().as_secs_f64() * 1e3;
let gpu_elapsed_ms = completed.gpu_execution_time();
diff --git a/crates/uzu/tests/unit/kernel/rope_test.rs b/crates/uzu/tests/unit/kernel/rope_test.rs
index 114d55fd7..24a306243 100644
--- a/crates/uzu/tests/unit/kernel/rope_test.rs
+++ b/crates/uzu/tests/unit/kernel/rope_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::RopeKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::RopeKernel},
cpu::Cpu,
},
};
@@ -93,7 +90,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let rotated_queries_array = context.create_array_uninitialized(&[queries_len], T::data_type(), "");
let rotated_keys_array = context.create_array_uninitialized(&[keys_len], T::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
qkv_array.buffer().borrow().deref(),
cosines_array.buffer().borrow().deref(),
@@ -107,9 +104,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
input.num_groups,
input.suffix_length,
input.max_sequence_length,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
(rotated_queries_array.as_slice().to_vec(), rotated_keys_array.as_slice().to_vec())
}
diff --git a/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs b/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs
index b64f31870..35c3f92cf 100644
--- a/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs
@@ -9,8 +9,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- Kernels,
+ Backend, Context, Encoder, Kernels,
gpu_types::ArgmaxPair,
kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},
},
@@ -71,16 +70,16 @@ fn get_output_single<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> V
let logits_array = context.create_array_from(&[len], &input.logits, "");
let output_array = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32, "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
logits_array.buffer().borrow().deref(),
output_array.buffer().borrow_mut().deref_mut(),
input.batch_size,
input.vocab_size,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice::<u32>().to_vec()
}
@@ -106,23 +105,23 @@ fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) ->
.create_buffer(partial_results_count * size_of::<ArgmaxPair>())
.expect("Failed to create partial results buffer");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
main_kernel.encode(
logits_array.buffer().borrow().deref(),
&mut partial_results_buffer,
input.batch_size,
input.vocab_size,
- &mut command_buffer,
+ &mut encoder,
);
final_kernel.encode(
&partial_results_buffer,
output_array.buffer().borrow_mut().deref_mut(),
input.batch_size,
input.vocab_size,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice::<u32>().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs b/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs
index 4fbdce850..5e293cd2a 100644
--- a/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::GumbelKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::GumbelKernel},
cpu::Cpu,
},
gumbel_float, revidx,
@@ -71,17 +68,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
false => context.create_array_uninitialized(&[len], T::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
logits_buffer,
seeds_array.buffer().borrow().deref(),
output_array.buffer().borrow_mut().deref_mut(),
input.batch_size,
input.vocab_size,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs b/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs
index 77f2c4944..d8d502c23 100644
--- a/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs
@@ -8,10 +8,7 @@ use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::TemperatureKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::TemperatureKernel},
cpu::Cpu,
},
};
@@ -68,17 +65,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
false => context.create_array_uninitialized(&[len], T::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
logits_buffer,
output_array.buffer().borrow_mut().deref_mut(),
input.batch_size,
input.vocab_size,
input.temperature,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs
index dfbbcb86d..993664f96 100644
--- a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs
@@ -7,8 +7,7 @@ use rand_distr::Normal;
use uzu::{
ArrayElement,
backends::common::{
- Backend, Buffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels,
+ Backend, Buffer, Context, Encoder, Kernels,
gpu_types::ArgmaxPair,
kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},
},
@@ -55,29 +54,23 @@ fn do_argmax_backend<B: Backend, T: ArrayElement + Float>(
let twopass_kernel_main = <B::Kernels as Kernels>::ArgmaxMainKernel::new(context, T::data_type()).unwrap();
let twopass_kernel_final = <B::Kernels as Kernels>::ArgmaxFinalKernel::new(context).unwrap();
- let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();
- single_kernel.encode(
- &logits_buffer,
- &mut single_output_buffer,
- batch_size as u32,
- vocab_size as u32,
- &mut command_buffer,
- );
+ let mut encoder = Encoder::new(context).unwrap();
+ single_kernel.encode(&logits_buffer, &mut single_output_buffer, batch_size as u32, vocab_size as u32, &mut encoder);
twopass_kernel_main.encode(
&logits_buffer,
&mut twopass_partial_results_buffer,
batch_size as u32,
vocab_size as u32,
- &mut command_buffer,
+ &mut encoder,
);
twopass_kernel_final.encode(
&twopass_partial_results_buffer,
&mut twopass_output_buffer,
batch_size as u32,
vocab_size as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let single_output = unsafe {
std::slice::from_raw_parts(single_output_buffer.cpu_ptr().as_ptr() as *const u32, batch_size)
diff --git a/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs
index 39aaad087..b8ef4d709 100644
--- a/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs
@@ -9,10 +9,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
- common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::TopKKernel,
- },
+ common::{Backend, Context, Encoder, Kernels, kernel::TopKKernel},
cpu::Cpu,
},
};
@@ -68,17 +65,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
false => context.create_array_uninitialized(&[len], T::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
logits_buffer,
output_array.buffer().borrow_mut().deref_mut(),
input.batch_size,
input.vocab_size,
input.top_k,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
index 8cabf92ad..0e6a47fd5 100644
--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
@@ -11,8 +11,7 @@ use uzu::{
ArrayContextExt, ArrayElement, DataType,
backends::{
common::{
- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,
- CommandBufferPending, Context, Kernels,
+ Backend, Context, Encoder, Kernels,
kernel::{TopPKernel, sampling::SamplingKernel},
},
cpu::Cpu,
@@ -73,17 +72,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
false => context.create_array_uninitialized(&[len], T::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
logits_buffer,
output_array.buffer().borrow_mut().deref_mut(),
input.batch_size,
input.vocab_size,
input.top_p,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
@@ -280,8 +279,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];
let seeds_array = context.create_array_from(&[batch_size], &seeds, "");
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
logits_array.buffer().borrow_mut().deref_mut(),
@@ -298,10 +296,10 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
},
batch_size,
vocab_size,
- &mut command_buffer,
+ &mut encoder,
)
.expect("encode");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let sampled_ids: &[u32] = output_array.as_slice();
@@ -384,8 +382,7 @@ fn test_topp_sampling_statistical_large() {
let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];
let seeds_array = context.create_array_from(&[BATCH], &seeds, "");
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
@@ -403,10 +400,10 @@ fn test_topp_sampling_statistical_large() {
},
BATCH,
VOCAB,
- &mut command_buffer,
+ &mut encoder,
)
.expect("encode");
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let sample_ids: &[u32] = output_array.as_slice();
for (b, &tok) in sample_ids.iter().enumerate() {
@@ -461,8 +458,7 @@ fn perf_topp_128k_vocab() {
let seeds_array = context.create_array_from(&[BATCH], &seeds, "");
let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");
- let mut command_buffer =
- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel
.encode(
@@ -480,12 +476,12 @@ fn perf_topp_128k_vocab() {
},
BATCH,
VOCAB,
- &mut command_buffer,
+ &mut encoder,
)
.expect("encode");
let host_timer = std::time::Instant::now();
- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
match completed.gpu_execution_time() {
diff --git a/crates/uzu/tests/unit/kernel/ssd_update_test.rs b/crates/uzu/tests/unit/kernel/ssd_update_test.rs
index b057715b8..84c885c75 100644
--- a/crates/uzu/tests/unit/kernel/ssd_update_test.rs
+++ b/crates/uzu/tests/unit/kernel/ssd_update_test.rs
@@ -7,8 +7,7 @@ use uzu::{
DataType,
backends::{
common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::SSDUpdateKernel,
+ Backend, Context, Encoder, Kernels, kernel::SSDUpdateKernel,
},
metal::Metal,
},
@@ -242,7 +241,7 @@ fn ssd_update_with_z_bf16() {
.expect("Failed to create buffer");
let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16, false).unwrap();
- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();
+ let mut encoder = Encoder::new(ctx.as_ref()).unwrap();
kernel.encode(
&x_buf,
&dt_buf,
@@ -262,9 +261,9 @@ fn ssd_update_with_z_bf16() {
bsz as u32,
h as u32,
dh as u32,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let y_ptr = y_buf.contents().as_ptr() as *const bf16;
let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };
diff --git a/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs
index c77ce3f45..0b90c6239 100644
--- a/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs
+++ b/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs
@@ -7,10 +7,7 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement,
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- Kernels, kernel::TensorAddBiasKernel,
- },
+ backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},
};
struct Input<T: ArrayElement + Float> {
@@ -77,16 +74,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
false => context.create_array_uninitialized(&[size], T::data_type(), ""),
};
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
input_buffer,
bias_array.buffer().borrow_mut().deref(),
output_array.buffer().borrow_mut().deref_mut(),
input.num_cols,
input.length,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
output_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs
index 10a880822..f1179f9e3 100644
--- a/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs
+++ b/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs
@@ -4,10 +4,7 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement,
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- Kernels, kernel::TensorAddSwapKernel,
- },
+ backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddSwapKernel},
};
struct Input<T: ArrayElement + Float> {
@@ -48,14 +45,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,
let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");
let main_array = context.create_array_from(&[size], &input.main_buffer, "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
skip_array.buffer().borrow_mut().deref_mut(),
main_array.buffer().borrow_mut().deref_mut(),
input.length,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
(skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())
}
diff --git a/crates/uzu/tests/unit/kernel/tensor_copy_test.rs b/crates/uzu/tests/unit/kernel/tensor_copy_test.rs
index 2af7a1929..5fb070f58 100644
--- a/crates/uzu/tests/unit/kernel/tensor_copy_test.rs
+++ b/crates/uzu/tests/unit/kernel/tensor_copy_test.rs
@@ -7,10 +7,7 @@ use half::{bf16, f16};
use num_traits::Float;
use uzu::{
ArrayContextExt, ArrayElement,
- backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- Kernels, kernel::TensorCopyKernel,
- },
+ backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorCopyKernel},
};
struct Input<T: ArrayElement + Float> {
@@ -47,14 +44,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let src_array = context.create_array_from(&[size], &input.src, "");
let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
kernel.encode(
src_array.buffer().borrow().deref(),
dst_array.buffer().borrow_mut().deref_mut(),
input.length,
- &mut command_buffer,
+ &mut encoder,
);
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
dst_array.as_slice().to_vec()
}
diff --git a/crates/uzu/tests/unit/kernel/token_copy_test.rs b/crates/uzu/tests/unit/kernel/token_copy_test.rs
index d64315950..7ef483573 100644
--- a/crates/uzu/tests/unit/kernel/token_copy_test.rs
+++ b/crates/uzu/tests/unit/kernel/token_copy_test.rs
@@ -3,8 +3,7 @@ use std::ops::{Deref, DerefMut};
use uzu::{
ArrayContextExt, ArrayElement,
backends::common::{
- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
- Kernels,
+ Backend, Context, Encoder, Kernels,
kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},
},
};
@@ -18,13 +17,9 @@ fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {
let src_array = context.create_array_from(&[1], &[src_value], "");
let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(
- src_array.buffer().borrow().deref(),
- dst_array.buffer().borrow_mut().deref_mut(),
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let output = dst_array.as_slice::<u64>().to_vec();
assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());
@@ -39,13 +34,9 @@ fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {
let src_array = context.create_array_from(&[1], &[src_value], "");
let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");
- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
- kernel.encode(
- src_array.buffer().borrow().deref(),
- dst_array.buffer().borrow_mut().deref_mut(),
- &mut command_buffer,
- );
- command_buffer.end_encoding().submit().wait_until_completed().unwrap();
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
let output = dst_array.as_slice::<u32>().to_vec();
assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());
diff --git a/crates/uzu/tests_unit/hazard_tracker_test.rs b/crates/uzu/tests_unit/hazard_tracker_test.rs
new file mode 100644
index 000000000..b3d51d89b
--- /dev/null
+++ b/crates/uzu/tests_unit/hazard_tracker_test.rs
@@ -0,0 +1,46 @@
+use super::*;
+
+#[test]
+fn test_hazard_tracker_simple_compute_sequence() {
+ let mut ht = HazardTracker::new();
+
+ // RAR is fine (many readers don't interfere with each other)
+ for _ in 0..123 {
+ assert_eq!(
+ ht.access(&[Access {
+ range: 0..16,
+ flags: AccessFlags::compute_read()
+ }]),
+ None
+ );
+ }
+
+ // WAR causes barrier after reads before writes (to ensure that the reads are done before writes can rewrite the data)
+ assert_eq!(
+ ht.access(&[Access {
+ range: 0..16,
+ flags: AccessFlags::compute_write()
+ }]),
+ Some((AccessFlags::compute_read(), AccessFlags::compute_write()))
+ );
+
+ // WAW causes barrier after writes before writes (to ensure the last write wins)
+ for _ in 0..123 {
+ assert_eq!(
+ ht.access(&[Access {
+ range: 0..16,
+ flags: AccessFlags::compute_write()
+ }]),
+ Some((AccessFlags::compute_write(), AccessFlags::compute_write()))
+ );
+ }
+
+ // RAW causes barrier after writes before reads (to ensure that writes wrote the data before reads read it)
+ assert_eq!(
+ ht.access(&[Access {
+ range: 0..16,
+ flags: AccessFlags::compute_read()
+ }]),
+ Some((AccessFlags::compute_write(), AccessFlags::compute_read()))
+ );
+}