A high-performance inference engine for AI models
basic hazard tracker and memory allocator (#269)
uuuvn33fa7019bde6
diff --git a/Cargo.lock b/Cargo.lockindex 99d5be237..20b9c3509 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2956,6 +2956,12 @@ dependencies = ["rand_core 0.9.5",]+[[package]]+name = "rangemap"+version = "1.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"+[[package]]name = "rawpointer"version = "0.2.1"@@ -4361,6 +4367,7 @@ dependencies = ["quote","rand 0.10.0","rand_distr",+ "rangemap","regex","schemars","serde",diff --git a/Cargo.toml b/Cargo.tomlindex e754234c3..d09f1bb1b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -50,6 +50,7 @@ walkdir = "2.5"depfile = "0.1"tempfile = "3.24"clang-ast = "0.1"+rangemap = "1.7.1"futures = "0.3"tokio = { version = "1.49", features = ["full"] }blake3 = "1.8"diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex a2da1559f..791bfc9dc 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -13,6 +13,7 @@ serde.workspace = trueserde_json.workspace = truethiserror.workspace = truetokenizers.workspace = true+rangemap.workspace = truebytemuck.workspace = truezerocopy = { workspace = true, features = ["derive"] }xxhash-rust = { workspace = true, features = ["xxh3"] }diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rsindex 6b0e8fd4a..1c110d652 100644--- a/crates/uzu/build/common/traitgen.rs+++ b/crates/uzu/build/common/traitgen.rs@@ -76,7 +76,7 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {fn new(context: &<Self::Backend as crate::backends::common::Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as crate::backends::common::Backend>::Error>;- fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#args ,)* command_buffer: &'command_buffer mut <<Self::Backend as crate::backends::common::Backend>::CommandBuffer as crate::backends::common::CommandBuffer>::Encoding);+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder mut crate::backends::common::Encoder<Self::Backend>);}};@@ -123,25 +123,53 @@ pub fn traitgen_all(backends_kernels: Vec<HashMap<Box<[Box<str>]>, Box<[Kernel]>}}+ impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::Buffer> for &'a crate::backends::common::Allocation<B> {+ fn into_parts(self) -> (&'a B::Buffer, usize) {+ let (buffer, range) = self.as_buffer_range();+ (buffer, range.start)+ }+ }+impl<'a, B: crate::backends::common::Buffer> BufferArg<'a, B> for (&'a B, usize) {fn into_parts(self) -> (&'a B, usize) {self}}+ impl<'a, B: crate::backends::common::Backend> BufferArg<'a, B::Buffer> for (&'a crate::backends::common::Allocation<B>, usize) {+ fn into_parts(self) -> (&'a B::Buffer, usize) {+ let (buffer, range) = self.0.as_buffer_range();+ (buffer, range.start + self.1)+ }+ }+pub trait BufferArgMut<'a, B: crate::backends::common::Buffer> {- fn into_parts(self) -> (&'a mut B, usize);+ fn into_parts(self) -> (&'a B, usize);}impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for &'a mut B {- fn into_parts(self) -> (&'a mut B, usize) {+ fn into_parts(self) -> (&'a B, usize) {(self, 0)}}+ impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::Buffer> for &'a mut crate::backends::common::Allocation<B> {+ fn into_parts(self) -> (&'a B::Buffer, usize) {+ let (buffer, range) = self.as_buffer_range();+ (buffer, range.start)+ }+ }+impl<'a, B: crate::backends::common::Buffer> BufferArgMut<'a, B> for (&'a mut B, usize) {- fn into_parts(self) -> (&'a mut B, usize) {- self+ fn into_parts(self) -> (&'a B, usize) {+ (&*self.0, self.1)+ }+ }++ impl<'a, B: crate::backends::common::Backend> BufferArgMut<'a, B::Buffer> for (&'a mut crate::backends::common::Allocation<B>, usize) {+ fn into_parts(self) -> (&'a B::Buffer, usize) {+ let (buffer, range) = self.0.as_buffer_range();+ (buffer, range.start + self.1)}}diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rsindex da8c99615..4fd6b0555 100644--- a/crates/uzu/build/cpu/compiler.rs+++ b/crates/uzu/build/cpu/compiler.rs@@ -340,8 +340,8 @@ impl CpuCompiler {(Some(quote! { #buffer_lifetime }),match access {- KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, Box<[u8]>> },- KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, Box<[u8]>> },+ KernelBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },+ KernelBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, std::cell::UnsafeCell<Box<[u8]>>> },},)},@@ -371,9 +371,9 @@ impl CpuCompiler {let argument_ident = &argument.name;match &argument.ty {FunctionArgumentType::Buffer(access) => {- let as_ptr_fn = match access {- KernelBufferAccess::Read => quote! { as_ptr },- KernelBufferAccess::ReadWrite => quote! { as_mut_ptr },+ let buffer_ptr = match access {+ KernelBufferAccess::Read => quote! { (&*__dsl_buffer.get()).as_ptr() },+ KernelBufferAccess::ReadWrite => quote! { (&mut *__dsl_buffer.get()).as_mut_ptr() },};if argument.conditional.is_some() {@@ -381,7 +381,7 @@ impl CpuCompiler {let #argument_ident = #argument_ident.map(|__dsl_buffer_impl| unsafe {let (__dsl_buffer, __dsl_offset) = __dsl_buffer_impl.into_parts();- __dsl_buffer.#as_ptr_fn().byte_add(__dsl_offset)+ #buffer_ptr.byte_add(__dsl_offset)});})} else {@@ -389,7 +389,7 @@ impl CpuCompiler {let #argument_ident = unsafe {let (__dsl_buffer, __dsl_offset) = #argument_ident.into_parts();- __dsl_buffer.#as_ptr_fn().byte_add(__dsl_offset)+ #buffer_ptr.byte_add(__dsl_offset)};})}@@ -439,7 +439,7 @@ impl CpuCompiler {});quote! {- encoder.push_command(move || #monomorphized_function(#function_call_args_joined));+ encoder.as_command_buffer_mut().push_command(move || #monomorphized_function(#function_call_args_joined));}};@@ -530,7 +530,7 @@ impl CpuCompiler {})}- fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer) {+ fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::common::Encoder<crate::backends::cpu::Cpu>) {#(#argument_copies)*#encode_body}diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex 528554320..ecf017cfa 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -99,11 +99,13 @@ pub fn bindgen(let mut arg_count: usize = 0;let mut indirect_flag = false;- let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_args_sets): (+ let (conditional_buffer_fields,conditional_buffer_sets, encode_generics, encode_args_defs, encode_deconstructs, encode_accesses, encode_args_sets): (Vec<Option<TokenStream>>,Vec<Option<TokenStream>>,Vec<Option<TokenStream>>,Vec<TokenStream>,+ Vec<Option<TokenStream>>,+ Vec<Option<TokenStream>>,Vec<TokenStream>,) = kernel.arguments@@ -113,7 +115,7 @@ pub fn bindgen(match ka.argument_type().unwrap() {arg_type @ (MetalArgumentType::Buffer(_) | MetalArgumentType::Constant(_)) => {- let (mut ty, mut set, generic) = match arg_type {+ let (mut ty, deconstruct, access_type, mut set, generic) = match arg_type {MetalArgumentType::Buffer(access) => {let buffer_lifetime = Lifetime::new(&format!("'{}", ka.name.as_ref()), Span::call_site());(@@ -121,9 +123,18 @@ pub fn bindgen(MetalBufferAccess::Read => quote! { impl crate::backends::common::kernel::BufferArg<#buffer_lifetime, Retained<ProtocolObject<dyn MTLBuffer>>> },MetalBufferAccess::ReadWrite => quote! { impl crate::backends::common::kernel::BufferArgMut<#buffer_lifetime, Retained<ProtocolObject<dyn MTLBuffer>>> },},+ Some(if ka.argument_condition().unwrap().is_some() {+ quote! {+ let #arg_name = #arg_name.map(|#arg_name| #arg_name.into_parts());+ }+ } else {+ quote! {+ let #arg_name = #arg_name.into_parts();+ }+ }),+ Some(access),quote! {- let (__dsl_buffer, __dsl_offset) = #arg_name.into_parts();- compute_encoder.set_buffer(Some(__dsl_buffer), __dsl_offset, #arg_count);+ compute_encoder.set_buffer(Some(#arg_name.0), #arg_name.1, #arg_count);},Some(quote! { #buffer_lifetime }),)@@ -133,11 +144,15 @@ pub fn bindgen(match constant_type {MetalConstantType::Scalar => (quote! { #arg_dtype },+ None,+ None,quote! { compute_encoder.set_value(&#arg_name, #arg_count); },None,),MetalConstantType::Array => (quote! { &[#arg_dtype] },+ None,+ None,quote! { compute_encoder.set_slice(#arg_name, #arg_count); },None,),@@ -146,12 +161,29 @@ pub fn bindgen(_ => unreachable!(),};+ let mut access = access_type.map(|access| {+ let compute_write = matches!(access, MetalBufferAccess::ReadWrite);++ quote! {+ crate::backends::common::Access {+ range: #arg_name.0.gpu_address_subrange((#arg_name.1)..(#arg_name.0.length())),+ flags: crate::backends::common::AccessFlags {+ compute_read: true,+ compute_write: #compute_write,+ copy_read: false,+ copy_write: false,+ },+ }+ }+ });+let (conditional_buffer_field, conditional_buffer_set) =if let Some(condition) = ka.argument_condition().unwrap() {let conditional_field_name = format_ident!("has_{}", ka.name.as_ref());let condition = parse_expr(condition.as_ref()).unwrap();ty = quote! { Option<#ty> };+ access = access.map(|access| quote! { #arg_name.as_ref().map(|#arg_name| #access)});set = quote! {assert!(#arg_name.is_some() == (self.#conditional_field_name));if let Some(#arg_name) = #arg_name {@@ -164,6 +196,8 @@ pub fn bindgen(Some(quote! { #conditional_field_name: #condition }),)} else {+ access = access.map(|access| quote! { Some(#access)});+(None, None)};@@ -174,13 +208,28 @@ pub fn bindgen(conditional_buffer_set,generic,quote! { #arg_name: #ty },+ deconstruct,+ access,set,))}MetalArgumentType::Groups(MetalGroupsType::Indirect) if !indirect_flag => {indirect_flag = true;- Some((None, None, Some(quote! { '__dsl_indirect_dispatch_buffer }), quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> }, quote! {}))+ Some((+ None,+ None,+ Some(quote! { '__dsl_indirect_dispatch_buffer }),+ quote! { __dsl_indirect_dispatch_buffer: impl crate::backends::common::kernel::BufferArg<'__dsl_indirect_dispatch_buffer, Retained<ProtocolObject<dyn MTLBuffer>>> },+ Some(quote! { let __dsl_indirect_dispatch_buffer = __dsl_indirect_dispatch_buffer.into_parts(); }),+ Some(quote! {+ Some(crate::backends::common::Access {+ range: __dsl_indirect_dispatch_buffer.0.gpu_address_subrange((__dsl_indirect_dispatch_buffer.1)..(__dsl_indirect_dispatch_buffer.1+12)),+ flags: crate::backends::common::AccessFlags::compute_read(),+ })+ }),+ quote! {},+ ))}_ => None,}@@ -191,6 +240,22 @@ pub fn bindgen(let conditional_buffer_sets = conditional_buffer_sets.into_iter().flatten().collect::<Vec<_>>();let encode_generics = encode_generics.into_iter().flatten().collect::<Vec<_>>();+ let encode_deconstructs = encode_deconstructs.into_iter().flatten().collect::<Vec<_>>();++ let encode_accesses = encode_accesses.into_iter().flatten().collect::<Vec<_>>();+ let encode_accesses = if !encode_accesses.is_empty() {+ let encode_accesses = encode_accesses.iter().fold(quote! {}, |a, b| {+ if !a.is_empty() && !b.is_empty() {+ quote! {#a , #b}+ } else {+ quote! {#a #b}+ }+ });+ quote! { encoder.access(&[#encode_accesses].into_iter().flatten().collect::<Vec<_>>()); }+ } else {+ quote! {}+ };+let (dispatch, elements) = if kernel.has_axis() {if kernel.has_groups() || kernel.has_threads() {anyhow::bail!("mixing groups/threads and axis is not supported");@@ -245,10 +310,9 @@ pub fn bindgen((quote! {- let (__dsl_buffer, __dsl_offset) = __dsl_indirect_dispatch_buffer.into_parts();compute_encoder.dispatch_threadgroups_indirect(- __dsl_buffer,- __dsl_offset,+ __dsl_indirect_dispatch_buffer.0,+ __dsl_indirect_dispatch_buffer.1,MTLSize::new(#((#threads) as usize, )*),);},@@ -337,9 +401,11 @@ pub fn bindgen(Ok(Self { pipeline #(, #conditional_buffer_sets)* })}- #method_visibility fn encode<#(#encode_generics, )* 'command_buffer>(&self, #(#encode_args_defs, )* command_buffer: &'command_buffer mut MetalCommandBufferEncoding) {- let compute_encoder = command_buffer.ensure_compute();+ #method_visibility fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::common::Encoder<crate::backends::metal::Metal>) {#empty_dispatch_guards+ #(#encode_deconstructs)*+ #encode_accesses+ let compute_encoder = encoder.as_command_buffer_mut().ensure_compute();compute_encoder.set_compute_pipeline_state(&self.pipeline);#(#encode_args_sets)*#dispatchdiff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rsindex cfafd4d6a..ae3db74d6 100644--- a/crates/uzu/build/metal/compiler.rs+++ b/crates/uzu/build/metal/compiler.rs@@ -280,9 +280,9 @@ impl MetalCompiler {use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};use objc2::{rc::Retained, runtime::ProtocolObject};+ use crate::backends::common::BufferGpuAddressRangeExt;use crate::backends::metal::{context::MetalContext,- command_buffer::MetalCommandBufferEncoding,error::MetalError,metal_extensions::{ComputeEncoderSetValue, FunctionConstantValuesSetValue, MetalDataTypeExt,diff --git a/crates/uzu/src/backends/common/allocator/allocator.rs b/crates/uzu/src/backends/common/allocator/allocator.rsnew file mode 100644index 000000000..093f37a37--- /dev/null+++ b/crates/uzu/src/backends/common/allocator/allocator.rs@@ -0,0 +1,197 @@+use std::{+ cell::RefCell,+ ops::Range,+ pin::Pin,+ rc::{Rc, Weak},+};++use super::{RangeAllocationType, RangeAllocator};+use crate::backends::common::{Backend, Buffer, Context};++pub struct Allocation<B: Backend> {+ allocator: Rc<Allocator<B>>,+ buffer: *const B::Buffer,+ range: Range<usize>,+}++impl<B: Backend> Allocation<B> {+ pub fn as_buffer_range<'a>(&'a self) -> (&'a B::Buffer, Range<usize>) {+ (unsafe { &*self.buffer }, self.range.clone())+ }+}++impl<B: Backend> Drop for Allocation<B> {+ fn drop(&mut self) {+ self.allocator.free(self)+ }+}++pub struct AllocationPool<B: Backend> {+ reusable: bool,+ allocator: Rc<Allocator<B>>,+ pool_number: usize,+}++impl<B: Backend> Drop for AllocationPool<B> {+ fn drop(&mut self) {+ self.allocator.free_pool(self)+ }+}++pub enum AllocationType<'a, B: Backend> {+ Global,+ Pooled {+ pool: &'a AllocationPool<B>,+ cpu_available: bool,+ },+}++struct AllocatorBuffer<B: Backend> {+ buffer: Pin<Box<B::Buffer>>,+ range_allocator: RangeAllocator<usize>,+}++pub struct Allocator<B: Backend> {+ context: Weak<B::Context>,+ allocator_buffers: RefCell<Vec<AllocatorBuffer<B>>>,+ next_pool_number: RefCell<usize>,+ peak_memory_usage: RefCell<usize>,+}++impl<B: Backend> Allocator<B> {+ pub fn new(context: Weak<B::Context>) -> Rc<Self> {+ Rc::new(Self {+ context,+ allocator_buffers: RefCell::new(Vec::new()),+ next_pool_number: RefCell::new(0),+ peak_memory_usage: RefCell::new(0),+ })+ }++ pub fn allocate(+ self: &Rc<Self>,+ size: usize,+ allocation_type: AllocationType<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let alignment = usize::clamp(size.next_power_of_two(), B::MIN_ALLOCATION_ALIGNMENT, 16_384);+ let allocation_type = match allocation_type {+ AllocationType::Global => RangeAllocationType::Global,+ AllocationType::Pooled {+ pool,+ cpu_available,+ } => RangeAllocationType::Pooled {+ pool: pool.pool_number,+ can_alias_before: !cpu_available,+ can_alias_after: !(cpu_available && pool.reusable),+ },+ };++ let mut allocator_buffers = self.allocator_buffers.borrow_mut();++ let mut found: Option<(*const B::Buffer, Range<usize>)> = None;++ for allocator_buffer in allocator_buffers.iter_mut() {+ if let Some(range) =+ allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type.clone())+ {+ found = Some((allocator_buffer.buffer.as_ref().get_ref() as *const B::Buffer, range));+ break;+ }+ }++ let (buffer, range) = if let Some((buffer, range)) = found {+ (buffer, range)+ } else {+ let new_allocator_buffer_size = usize::max(size, 268_435_456);++ let mut allocator_buffer = AllocatorBuffer::<B> {+ buffer: Box::pin(self.context.upgrade().unwrap().create_buffer(new_allocator_buffer_size)?), // Upgrade can never fail+ range_allocator: RangeAllocator::new(0..new_allocator_buffer_size),+ };++ let buffer = allocator_buffer.buffer.as_ref().get_ref() as *const B::Buffer;+ let range =+ allocator_buffer.range_allocator.allocate_range_aligned(size, alignment, allocation_type).unwrap(); // Can never fail++ allocator_buffers.push(allocator_buffer);++ *self.peak_memory_usage.borrow_mut() =+ allocator_buffers.iter().map(|allocator_buffer| allocator_buffer.buffer.length()).sum();++ (buffer, range)+ };++ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());++ Ok(Allocation {+ allocator: self.clone(),+ buffer,+ range,+ })+ }++ pub fn create_pool(+ self: &Rc<Self>,+ reusable: bool,+ ) -> AllocationPool<B> {+ let pool_number = *self.next_pool_number.borrow();++ *self.next_pool_number.borrow_mut() += 1;++ AllocationPool {+ reusable,+ allocator: self.clone(),+ pool_number,+ }+ }++ pub fn peak_memory_usage(self: Rc<Self>) -> usize {+ *self.peak_memory_usage.borrow()+ }++ // TODO: Maybe hysteresis in free/free_pool?++ fn free(+ self: &Rc<Self>,+ allocation: &Allocation<B>,+ ) {+ let mut allocator_buffers = self.allocator_buffers.borrow_mut();++ let (allocator_buffer_index, allocator_buffer) = allocator_buffers+ .iter_mut()+ .enumerate()+ .find(|(_allocator_buffer_index, allocator_buffer)| {+ (allocator_buffer.buffer.as_ref().get_ref() as *const B::Buffer) == allocation.buffer+ })+ .unwrap(); // Can never fail++ allocator_buffer.range_allocator.free_range(allocation.range.clone());++ if allocator_buffer.range_allocator.is_empty() {+ allocator_buffers.remove(allocator_buffer_index);+ }++ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());+ }++ fn free_pool(+ self: &Rc<Self>,+ pool: &AllocationPool<B>,+ ) {+ let mut allocator_buffers = self.allocator_buffers.borrow_mut();+ let mut free_idxs = Vec::new();++ for (allocation_buffer_idx, allocation_buffer) in allocator_buffers.iter_mut().enumerate() {+ allocation_buffer.range_allocator.free_pool(pool.pool_number);+ if allocation_buffer.range_allocator.is_empty() {+ free_idxs.push(allocation_buffer_idx);+ }+ }++ for (i, free_idx) in free_idxs.into_iter().enumerate() {+ allocator_buffers.remove(free_idx - i);+ }++ allocator_buffers.sort_by_key(|allocator_buffer| allocator_buffer.range_allocator.total_available());+ }+}diff --git a/crates/uzu/src/backends/common/allocator/mod.rs b/crates/uzu/src/backends/common/allocator/mod.rsnew file mode 100644index 000000000..c6e4a183c--- /dev/null+++ b/crates/uzu/src/backends/common/allocator/mod.rs@@ -0,0 +1,5 @@+mod allocator;+mod range_allocator;++pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};+use range_allocator::{AllocationType as RangeAllocationType, RangeAllocator};diff --git a/crates/uzu/src/backends/common/allocator/range_allocator.rs b/crates/uzu/src/backends/common/allocator/range_allocator.rsnew file mode 100644index 000000000..635729787--- /dev/null+++ b/crates/uzu/src/backends/common/allocator/range_allocator.rs@@ -0,0 +1,197 @@+use std::ops::Range;++use rangemap::{RangeMap, RangeSet};++#[derive(PartialEq, Eq, Clone)]+pub enum AllocationType<P> {+ Global,+ Pooled {+ pool: P,+ can_alias_before: bool,+ can_alias_after: bool,+ },+}++#[derive(PartialEq, Eq, Clone)]+enum PooledAllocationRangeStatus {+ Allocated {+ can_alias_after: bool,+ },+ Aliasable,+ Frozen,+}++#[derive(PartialEq, Eq, Clone)]+enum AllocationRangeType<P: Eq + Clone> {+ Global,+ Pooled {+ pool: P,+ status: PooledAllocationRangeStatus,+ },+}++pub struct RangeAllocator<P: Eq + Clone> {+ full_range: Range<usize>,+ ranges: RangeMap<usize, AllocationRangeType<P>>,+}++impl<P: Eq + Clone> RangeAllocator<P> {+ pub fn new(full_range: Range<usize>) -> Self {+ Self {+ full_range,+ ranges: RangeMap::new(),+ }+ }++ pub fn allocate_range_aligned(+ &mut self,+ size: usize,+ alignment: usize,+ allocation_type: AllocationType<P>,+ ) -> Option<Range<usize>> {+ let mut available_range = RangeSet::new();+ available_range.extend(self.ranges.gaps(&self.full_range));++ if let AllocationType::Pooled {+ pool,+ can_alias_before: true,+ can_alias_after: _,+ } = &allocation_type+ {+ available_range.extend(self.ranges.iter().flat_map(|(range, range_type)| {+ if let AllocationRangeType::Pooled {+ pool: range_pool,+ status: PooledAllocationRangeStatus::Aliasable,+ } = range_type+ && range_pool == pool+ {+ Some(range.clone())+ } else {+ None+ }+ }));+ }++ let Some(smallest_fitting_free_range) = available_range+ .iter()+ .filter(|range| range.end.saturating_sub(range.start.next_multiple_of(alignment)) >= size)+ .min_by_key(|range| range.len())+ else {+ return None;+ };++ let allocated_range_start = smallest_fitting_free_range.start.next_multiple_of(alignment);+ let allocated_range = allocated_range_start..(allocated_range_start + size);+ let allocated_range_status = match allocation_type {+ AllocationType::Global => AllocationRangeType::Global,+ AllocationType::Pooled {+ pool,+ can_alias_before: _,+ can_alias_after,+ } => AllocationRangeType::Pooled {+ pool,+ status: PooledAllocationRangeStatus::Allocated {+ can_alias_after,+ },+ },+ };++ self.ranges.insert(allocated_range.clone(), allocated_range_status);++ Some(allocated_range)+ }++ pub fn free_range(+ &mut self,+ range: Range<usize>,+ ) {+ let overlapping_ranges = self.ranges.overlapping(range.clone()).collect::<Vec<_>>();+ assert!(overlapping_ranges.len() == 1, "expected only 1 overlapping range");++ let (overlapping_range, range_type) = overlapping_ranges[0];+ assert!(+ overlapping_range.start <= range.start && overlapping_range.end >= range.end,+ "overlapping range doesn't fully cover the range"+ );++ match range_type {+ AllocationRangeType::Global => self.ranges.remove(range),+ AllocationRangeType::Pooled {+ pool,+ status,+ } => {+ let status = match status {+ PooledAllocationRangeStatus::Allocated {+ can_alias_after: true,+ } => PooledAllocationRangeStatus::Aliasable,+ PooledAllocationRangeStatus::Allocated {+ can_alias_after: false,+ } => PooledAllocationRangeStatus::Frozen,+ _ => panic!("attempted to free a range that isn't allocated"),+ };++ self.ranges.insert(+ range,+ AllocationRangeType::Pooled {+ pool: pool.clone(),+ status,+ },+ )+ },+ };+ }++ pub fn free_pool(+ &mut self,+ pool: P,+ ) {+ let pool_ranges = self+ .ranges+ .iter()+ .filter_map(|(range, range_type)| {+ if let AllocationRangeType::Pooled {+ pool: range_pool,+ status,+ } = range_type+ && range_pool == &pool+ {+ assert!(+ !matches!(status, PooledAllocationRangeStatus::Allocated { .. }),+ "attempted to free a pool that has live allocations"+ );+ Some(range.clone())+ } else {+ None+ }+ })+ .collect::<Vec<_>>();++ for pool_range in pool_ranges {+ self.ranges.remove(pool_range);+ }+ }++ pub fn total_available(&self) -> usize {+ self.ranges+ .gaps(&self.full_range)+ .map(|gap| gap.len())+ .chain(self.ranges.iter().filter_map(|(range, range_type)| {+ if matches!(+ range_type,+ AllocationRangeType::Pooled {+ pool: _,+ status: PooledAllocationRangeStatus::Aliasable+ }+ ) {+ Some(range.len())+ } else {+ None+ }+ }))+ .sum()+ }++ pub fn is_empty(&self) -> bool {+ self.ranges.is_empty()+ }+}diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rsindex e7bba72f3..e0f18747a 100644--- a/crates/uzu/src/backends/common/backend.rs+++ b/crates/uzu/src/backends/common/backend.rs@@ -10,5 +10,6 @@ pub trait Backend: Debug + Clone + 'static {type Kernels: Kernels<Backend = Self> + MatmulKernels;type Error: Error + Debug;+ const MIN_ALLOCATION_ALIGNMENT: usize;const MAX_INLINE_BYTES: usize;}diff --git a/crates/uzu/src/backends/common/buffer.rs b/crates/uzu/src/backends/common/buffer.rsindex 832ce2cc9..2a628617f 100644--- a/crates/uzu/src/backends/common/buffer.rs+++ b/crates/uzu/src/backends/common/buffer.rs@@ -1,4 +1,4 @@-use std::{fmt::Debug, os::raw::c_void, ptr::NonNull};+use std::{fmt::Debug, ops::Range, os::raw::c_void, ptr::NonNull};use super::Backend;@@ -11,8 +11,24 @@ pub trait Buffer: Debug {);fn cpu_ptr(&self) -> NonNull<c_void>;+ fn gpu_ptr(&self) -> usize;fn length(&self) -> usize;+}++pub trait BufferGpuAddressRangeExt: Buffer {+ fn gpu_address_range(&self) -> Range<usize> {+ self.gpu_ptr()..(self.gpu_ptr() + self.length())+ }- fn id(&self) -> usize;+ fn gpu_address_subrange(+ &self,+ subrange: Range<usize>,+ ) -> Range<usize> {+ assert!(subrange.end <= self.length(), "subrange overflow: subrange={:?} length={}", subrange, self.length());++ (self.gpu_ptr() + subrange.start)..(self.gpu_ptr() + subrange.end)+ }}++impl<B: Buffer> BufferGpuAddressRangeExt for B {}diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex d597c899b..523aff8cf 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -18,6 +18,55 @@ pub trait CommandBufferInitial {fn start_encoding(self) -> <Self::CommandBuffer as CommandBuffer>::Encoding;}+#[derive(Debug, Clone, PartialEq)]+pub struct AccessFlags {+ pub compute_read: bool,+ pub compute_write: bool,+ pub copy_read: bool,+ pub copy_write: bool,+}++impl AccessFlags {+ pub fn empty() -> Self {+ Self {+ compute_read: false,+ compute_write: false,+ copy_read: false,+ copy_write: false,+ }+ }++ pub fn with_compute_read(mut self) -> Self {+ self.compute_read = true;+ self+ }+ pub fn with_compute_write(mut self) -> Self {+ self.compute_write = true;+ self+ }+ pub fn with_copy_read(mut self) -> Self {+ self.copy_read = true;+ self+ }+ pub fn with_copy_write(mut self) -> Self {+ self.copy_write = true;+ self+ }++ pub fn compute_read() -> Self {+ Self::empty().with_compute_read()+ }+ pub fn compute_write() -> Self {+ Self::empty().with_compute_write()+ }+ pub fn copy_read() -> Self {+ Self::empty().with_copy_read()+ }+ pub fn copy_write() -> Self {+ Self::empty().with_copy_write()+ }+}+pub trait CommandBufferEncoding {type CommandBuffer: CommandBuffer<Encoding = Self>;@@ -35,6 +84,12 @@ pub trait CommandBufferEncoding {value: u8,);+ fn encode_barrier(+ &mut self,+ after: AccessFlags,+ before: AccessFlags,+ );+fn encode_wait_for_event(&mut self,event: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Event,diff --git a/crates/uzu/src/backends/common/context.rs b/crates/uzu/src/backends/common/context.rsindex 6daca784b..ddcd74db1 100644--- a/crates/uzu/src/backends/common/context.rs+++ b/crates/uzu/src/backends/common/context.rs@@ -1,7 +1,6 @@use std::{env, path::Path, rc::Rc};-use super::Backend;-use crate::backends::common::CommandBuffer;+use crate::backends::common::{Allocation, AllocationPool, AllocationType, Backend, CommandBuffer};pub trait Context: Sized {type Backend: Backend<Context = Self>;@@ -26,6 +25,17 @@ pub trait Context: Sized {size: usize,) -> Result<<Self::Backend as Backend>::Buffer, <Self::Backend as Backend>::Error>;+ fn create_allocation(+ &self,+ size: usize,+ allocation_type: AllocationType<Self::Backend>,+ ) -> Result<Allocation<Self::Backend>, <Self::Backend as Backend>::Error>;++ fn create_allocation_pool(+ &self,+ reusable: bool,+ ) -> AllocationPool<Self::Backend>;+fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;fn enable_capture();diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rsnew file mode 100644index 000000000..7c0136060--- /dev/null+++ b/crates/uzu/src/backends/common/encoder.rs@@ -0,0 +1,173 @@+use std::{ops::Range, time::Duration};++use crate::backends::common::{+ AccessFlags, Allocation, AllocationPool, AllocationType, Backend, BufferGpuAddressRangeExt, CommandBuffer,+ CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,+ Context,+ hazard_tracker::{Access, HazardTracker},+};++pub struct Encoder<'encoding, B: Backend> {+ context: &'encoding B::Context,+ command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,+ allocation_pool: AllocationPool<B>,+ hazard_tracker: HazardTracker,+}++impl<'encoding, B: Backend> Encoder<'encoding, B> {+ pub fn new(context: &'encoding B::Context) -> Result<Self, B::Error> {+ let command_buffer = context.create_command_buffer()?.start_encoding();+ let allocation_pool = context.create_allocation_pool(false);+ let hazard_tracker = HazardTracker::new();++ Ok(Self {+ context,+ command_buffer,+ allocation_pool,+ hazard_tracker,+ })+ }++ // This is valid on both cpu and gpu timelines+ pub fn allocate_constant(+ &mut self,+ size: usize,+ ) -> Result<Allocation<B>, B::Error> {+ self.context.create_allocation(+ size,+ AllocationType::Pooled {+ pool: &self.allocation_pool,+ cpu_available: true,+ },+ )+ }++ // This is valid on gpu timeline only+ pub fn allocate_scratch(+ &mut self,+ size: usize,+ ) -> Result<Allocation<B>, B::Error> {+ self.context.create_allocation(+ size,+ AllocationType::Pooled {+ pool: &self.allocation_pool,+ cpu_available: false,+ },+ )+ }++ pub fn encode_copy(+ &mut self,+ src: &B::Buffer,+ dst: &mut B::Buffer,+ size: usize,+ ) {+ self.access(&[+ Access {+ range: src.gpu_address_subrange(0..size),+ flags: AccessFlags::copy_read(),+ },+ Access {+ range: dst.gpu_address_subrange(0..size),+ flags: AccessFlags::copy_write(),+ },+ ]);+ self.command_buffer.encode_copy(src, dst, size);+ }++ pub fn encode_fill(+ &mut self,+ dst: &mut B::Buffer,+ range: Range<usize>,+ value: u8,+ ) {+ self.access(&[Access {+ range: dst.gpu_address_subrange(range.clone()),+ flags: AccessFlags::copy_write(),+ }]);+ self.command_buffer.encode_fill(dst, range, value);+ }++ pub fn access(+ &mut self,+ accesses: &[Access],+ ) {+ if let Some((after, before)) = self.hazard_tracker.access(accesses) {+ self.command_buffer.encode_barrier(after, before);+ }+ }++ pub fn encode_wait_for_event(+ &mut self,+ event: &B::Event,+ value: u64,+ ) {+ self.command_buffer.encode_wait_for_event(event, value);+ }++ pub fn encode_signal_event(+ &mut self,+ event: &B::Event,+ value: u64,+ ) {+ self.command_buffer.encode_signal_event(event, value);+ }++ pub fn add_completion_handler(+ &mut self,+ handler: impl FnOnce(Result<&<B::CommandBuffer as CommandBuffer>::Completed, B::Error>) + 'static,+ ) {+ self.command_buffer.add_completion_handler(handler);+ }++ pub fn as_command_buffer_mut(&mut self) -> &mut <B::CommandBuffer as CommandBuffer>::Encoding {+ &mut self.command_buffer+ }++ pub fn end_encoding(self) -> Executable<B> {+ Executable {+ command_buffer: self.command_buffer.end_encoding(),+ allocation_pool: self.allocation_pool,+ }+ }+}++pub struct Executable<B: Backend> {+ command_buffer: <B::CommandBuffer as CommandBuffer>::Executable,+ allocation_pool: AllocationPool<B>,+}++impl<B: Backend> Executable<B> {+ pub fn submit(self) -> Pending<B> {+ Pending {+ command_buffer: self.command_buffer.submit(),+ allocation_pool: self.allocation_pool,+ }+ }+}++pub struct Pending<B: Backend> {+ command_buffer: <B::CommandBuffer as CommandBuffer>::Pending,+ allocation_pool: AllocationPool<B>,+}++impl<B: Backend> Pending<B> {+ pub fn wait_until_completed(self) -> Result<Completed<B>, B::Error> {+ Ok(Completed {+ command_buffer: self.command_buffer.wait_until_completed()?,+ allocation_pool: self.allocation_pool,+ })+ }+}++pub struct Completed<B: Backend> {+ command_buffer: <B::CommandBuffer as CommandBuffer>::Completed,+ #[allow(unused)]+ allocation_pool: AllocationPool<B>,+}++impl<B: Backend> Completed<B> {+ pub fn gpu_execution_time(&self) -> Option<Duration> {+ self.command_buffer.gpu_execution_time()+ }+}diff --git a/crates/uzu/src/backends/common/hazard_tracker.rs b/crates/uzu/src/backends/common/hazard_tracker.rsnew file mode 100644index 000000000..fc1ea6f4c--- /dev/null+++ b/crates/uzu/src/backends/common/hazard_tracker.rs@@ -0,0 +1,98 @@+use std::ops::Range;++use rangemap::RangeSet;++use crate::backends::common::AccessFlags;++#[derive(Debug, Clone)]+pub struct Access {+ pub range: Range<usize>,+ pub flags: AccessFlags,+}++pub struct HazardTracker {+ compute_reads: RangeSet<usize>,+ compute_writes: RangeSet<usize>,+ copy_reads: RangeSet<usize>,+ copy_writes: RangeSet<usize>,+}++impl HazardTracker {+ pub fn new() -> Self {+ Self {+ compute_reads: RangeSet::new(),+ compute_writes: RangeSet::new(),+ copy_reads: RangeSet::new(),+ copy_writes: RangeSet::new(),+ }+ }++ #[must_use]+ pub fn access(+ &mut self,+ accesses: &[Access],+ ) -> Option<(AccessFlags, AccessFlags)> {+ let mut after = AccessFlags::empty();+ let mut before = AccessFlags::empty();++ // Get existing hazards (before the barrier, aka barrier after this)+ for access in accesses {+ // Any access should wait for earlier overlapping writes to complete (RAW/WAW)+ if access.flags.compute_read+ || access.flags.compute_write+ || access.flags.copy_read+ || access.flags.copy_write+ {+ if self.compute_writes.overlaps(&access.range) {+ after.compute_write = true;+ self.compute_writes.clear();+ }+ if self.copy_writes.overlaps(&access.range) {+ after.copy_write = true;+ self.copy_writes.clear();+ }+ }+ // Write accesses also must wait for previous reads to complete (WAR)+ if access.flags.compute_write || access.flags.copy_write {+ if self.compute_reads.overlaps(&access.range) {+ after.compute_read = true;+ self.compute_reads.clear();+ }+ if self.copy_reads.overlaps(&access.range) {+ after.copy_read = true;+ self.copy_reads.clear();+ }+ }+ }++ // Track new ones (after the barrier, aka barrier before this)+ for access in accesses {+ if access.flags.compute_read {+ before.compute_read = true;+ self.compute_reads.insert(access.range.clone());+ }+ if access.flags.compute_write {+ before.compute_write = true;+ self.compute_writes.insert(access.range.clone());+ }+ if access.flags.copy_read {+ before.copy_read = true;+ self.copy_reads.insert(access.range.clone());+ }+ if access.flags.copy_write {+ before.copy_write = true;+ self.copy_writes.insert(access.range.clone());+ }+ }++ if after.compute_read || after.compute_write || after.copy_read || after.copy_write {+ Some((after, before))+ } else {+ None+ }+ }+}++#[cfg(test)]+#[path = "../../../tests_unit/hazard_tracker_test.rs"]+mod hazard_tracker_test;diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rsindex c0522bdd7..339d104fd 100644--- a/crates/uzu/src/backends/common/kernel/attention.rs+++ b/crates/uzu/src/backends/common/kernel/attention.rs@@ -6,7 +6,7 @@ use std::{use crate::{DataType,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,gpu_types::{AttnMaskParams, AttnParams},kernel::AttentionGemmKernel,},@@ -49,7 +49,7 @@ impl<B: Backend> AttentionGemmBlock<B> {pub fn encode(&self,context: &B::Context,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: AttentionGemmArguments<B>,) -> Result<(), B::Error> {let bk: usize = if args.head_dim < 128 {@@ -138,7 +138,7 @@ impl<B: Backend> AttentionGemmBlock<B> {args.sinks_buffer,args.num_heads as u32,args.suffix_length as u32,- command_buffer,+ encoder,);Ok(())diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rsindex 04692fa70..f7be333b6 100644--- a/crates/uzu/src/backends/common/kernel/audio.rs+++ b/crates/uzu/src/backends/common/kernel/audio.rs@@ -1,4 +1,4 @@-use crate::backends::common::{Backend, CommandBuffer};+use crate::backends::common::{Backend, Encoder};#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum AudioPadMode {@@ -120,68 +120,68 @@ pub struct AudioClampArguments<'a, B: Backend> {pub trait AudioKernelRuntime<B: Backend> {fn encode_fsq_decode(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioFsqDecodeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_fsq_encode(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioFsqEncodeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_conv1d(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioConv1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_causal_conv1d(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioCausalConv1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_causal_conv_transpose1d(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioCausalConvTranspose1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_half_snake(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioHalfSnakeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_leaky_relu(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioElementwiseArguments<'_, B>,negative_slope: f32,) -> Result<(), B::Error>;fn encode_tanh(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioElementwiseArguments<'_, B>,) -> Result<(), B::Error>;fn encode_add(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioAddArguments<'_, B>,) -> Result<(), B::Error>;fn encode_scale(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioScaleArguments<'_, B>,) -> Result<(), B::Error>;fn encode_clamp(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: AudioClampArguments<'_, B>,) -> Result<(), B::Error>;}diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsindex d208d7156..daedd5865 100644--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -4,7 +4,7 @@ use thiserror::Error;use crate::{DataType,- backends::common::{Backend, CommandBuffer, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},+ backends::common::{Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},};pub struct KVLayerData<B: Backend> {@@ -58,7 +58,7 @@ impl<B: Backend> KVCacheUpdate<B> {in_place_data: &[KVLayerData<B>],source_indices: &[usize],destination_indices: &[usize],- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), KVCacheUpdateError<B>> {if source_indices.len() != destination_indices.len() {return Err(KVCacheUpdateError::IndicesCountMismatch);@@ -87,7 +87,7 @@ impl<B: Backend> KVCacheUpdate<B> {num_heads as u32,max_sequence_length as u32,head_dim as u32,- command_buffer,+ encoder,);}}diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rsindex f1c793475..78a99f7d8 100644--- a/crates/uzu/src/backends/common/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs@@ -6,7 +6,7 @@ use thiserror::Error;use super::Kernels;use crate::{DataType,- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, Encoder},};pub trait MatmulKernels: Kernels {@@ -25,7 +25,7 @@ pub trait MatmulKernel: Sized {&mut self,context: &<Self::Backend as Backend>::Context,arguments: MatmulArguments<Self::Backend>,- command_buffer: &mut <<Self::Backend as Backend>::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<Self::Backend>,);}diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rsindex 2c5f1863a..0561ac39f 100644--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -1,7 +1,7 @@use crate::{DataType,backends::common::{- ActivationConfig, Backend, CommandBuffer, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,+ ActivationConfig, Backend, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,},};@@ -28,7 +28,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {pub fn encode(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,fused_up: &B::Buffer,hidden: &mut B::Buffer,m: i32,@@ -36,7 +36,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {if self.activation.act_type() == ActivationType::IDENTITY {panic!("Identity activation is not supported for kernel")}- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), command_buffer);+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), encoder);Ok(())}}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rsindex bcaa38a61..484cb916b 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs@@ -3,7 +3,7 @@ use std::ops::{Deref, DerefMut};use crate::{DataType,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeExpertsDecodeSinglePassAKernel, MoeExpertsDecodeSinglePassBKernel},},};@@ -84,7 +84,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {pub fn encode(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,mut args: MoeExpertsSingleDecodeArguments<B>,) {if args.k == 0 {@@ -110,7 +110,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {args.gate_clip_max,args.up_clip_min,args.up_clip_max,- command_buffer,+ encoder,);// Pass B: 8 simdgroups (256 threads), outputs final y directly@@ -125,7 +125,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {args.d_model as u32,args.d_ff as u32,args.k as u32,- command_buffer,+ encoder,);}}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rsindex 1030d6a5c..0749d385e 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs@@ -7,7 +7,7 @@ use super::{use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, MoeExpertsDecodeDownFused2DKernel, MoeExpertsDecodePassAKernel},},};@@ -47,7 +47,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {pub fn encode(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,mut args: MoeExpertsTwoPassArguments<B>,) {if args.total_rows == 0 {@@ -58,7 +58,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {const BLOCK_M: u32 = 4;let h_blocks = (args.d_ff as u32 + BLOCK_M - 1) / BLOCK_M;self.pass_a_tile.encode_counts(- command_buffer,+ encoder,MoePassATileCountsArguments {expert_offsets: args.expert_offsets,tile_counts: args.tile_counts.deref_mut(),@@ -67,7 +67,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {},);self.pass_a_tile.encode_scan(- command_buffer,+ encoder,MoePassATileScanArguments {tile_counts: args.tile_counts,tile_offsets: args.tile_offsets.deref_mut(),@@ -76,7 +76,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {},);self.pass_a_tile.encode_row_map(- command_buffer,+ encoder,MoePassARowMapArguments {expert_offsets: args.expert_offsets,row_expert_map: args.row_expert_map.deref_mut(),@@ -85,7 +85,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {},);self.pass_a_tile.encode_build_map(- command_buffer,+ encoder,MoePassATileBuildArguments {expert_offsets: args.expert_offsets,tile_offsets: args.tile_offsets,@@ -96,7 +96,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {},);self.pass_a_tile.encode_dispatch_args(- command_buffer,+ encoder,MoePassATileDispatchArguments {total_tiles: args.total_tiles,dispatch_args: args.dispatch_args.deref_mut(),@@ -125,7 +125,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {args.silu_alpha,args.tile_map.deref(),args.dispatch_args.deref(),- command_buffer,+ encoder,);// pass b@@ -140,7 +140,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {args.d_model as u32,args.d_ff as u32,args.e as u32,- command_buffer,+ encoder,);}}diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rsindex 15cd2e280..4cae93202 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs@@ -6,7 +6,7 @@ use super::{use crate::{DataType,backends::common::{- Backend, CommandBuffer, CommandBufferEncoding, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeExpertsPrefillPassAKernel, MoeExpertsPrefillPassBKernel},},};@@ -48,7 +48,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {pub fn encode(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,mut args: MoeExpertsTwoPassArguments<B>,) {if args.total_rows == 0 {@@ -56,10 +56,10 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {}let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();- command_buffer.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);+ encoder.encode_fill(args.hidden_buffer.deref_mut(), 0..hidden_bytes, 0);self.tile_map.encode_counts(- command_buffer,+ encoder,MoeTileCountsArguments {offsets_buffer: args.expert_offsets,tile_counts_buffer: args.tile_counts.deref_mut(),@@ -67,7 +67,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {},);self.tile_map.encode_scan(- command_buffer,+ encoder,MoeTileScanArguments {tile_counts_buffer: args.tile_counts,tile_offsets_buffer: args.tile_offsets.deref_mut(),@@ -76,7 +76,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {},);self.tile_map.encode_build_map(- command_buffer,+ encoder,MoeTileMapBuildArguments {expert_offsets: args.expert_offsets,tile_offsets: args.tile_offsets,@@ -100,7 +100,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {(d_model_u32 + COL_TILE_MODEL - 1) / COL_TILE_MODEL};self.tile_map.encode_dispatch_args(- command_buffer,+ encoder,MoeTileDispatchArguments {total_tiles: args.total_tiles,dispatch_args: args.dispatch_args.deref_mut(),@@ -128,7 +128,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args.silu_alpha,args.tile_map.deref(),args.dispatch_args.deref(),- command_buffer,+ encoder,);let dispatch_args = MoeTileDispatchArguments {@@ -136,7 +136,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {dispatch_args: args.dispatch_args.deref_mut(),num_tiles_x: n_tiles_model,};- self.tile_map.encode_dispatch_args(command_buffer, dispatch_args);+ self.tile_map.encode_dispatch_args(encoder, dispatch_args);let kernel_pass_b = &self.pass_b_indirect[dtype_idx];kernel_pass_b.encode(@@ -150,7 +150,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args.e as u32,args.tile_map.deref(),args.dispatch_args.deref(),- command_buffer,+ encoder,);}}diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rsindex 771c32ada..fc7c2cf8f 100644--- a/crates/uzu/src/backends/common/kernel/moe/gather.rs+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs@@ -1,7 +1,7 @@use crate::{DataType,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},},};@@ -34,7 +34,7 @@ impl<B: Backend> MoeGatherKernels<B> {pub fn encode(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,dtype: DataType,args: MoeGatherArguments<B>,) {@@ -47,7 +47,7 @@ impl<B: Backend> MoeGatherKernels<B> {args.d_model as u32,args.t as u32,args.k as u32,- command_buffer,+ encoder,),DataType::F16 => self.f16.encode(args.x_buffer,@@ -57,7 +57,7 @@ impl<B: Backend> MoeGatherKernels<B> {args.d_model as u32,args.t as u32,args.k as u32,- command_buffer,+ encoder,),DataType::BF16 => self.bf16.encode(args.x_buffer,@@ -67,7 +67,7 @@ impl<B: Backend> MoeGatherKernels<B> {args.d_model as u32,args.t as u32,args.k as u32,- command_buffer,+ encoder,),_ => panic!("Unsupported data type: {:?}", dtype),};diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rsindex ea144b317..e1c068c24 100644--- a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs@@ -1,5 +1,5 @@use crate::backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeBuildTileMapKernel, MoeTileCountsKernel, MoeTileScanKernel, MoeWriteDispatchArgsKernel},};@@ -22,15 +22,15 @@ impl<B: Backend> MoeTileMapKernels<B> {pub fn encode_counts(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoeTileCountsArguments<B>,) {- self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, command_buffer);+ self.counts.encode(args.offsets_buffer, args.tile_counts_buffer, args.e as u32, encoder);}pub fn encode_scan(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoeTileScanArguments<B>,) {self.scan.encode(@@ -38,13 +38,13 @@ impl<B: Backend> MoeTileMapKernels<B> {args.tile_offsets_buffer,args.total_tiles_buffer,args.e as u32,- command_buffer,+ encoder,);}pub fn encode_build_map(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoeTileMapBuildArguments<B>,) {self.build.encode(@@ -53,16 +53,16 @@ impl<B: Backend> MoeTileMapKernels<B> {args.tile_counts,args.tile_map,args.e as u32,- command_buffer,+ encoder,);}pub fn encode_dispatch_args(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoeTileDispatchArguments<B>,) {- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, command_buffer);+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_x, encoder);}}diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rsindex 22353c53c..69c70c669 100644--- a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs@@ -1,5 +1,5 @@use crate::backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, MoePassABuildRowMapKernel, MoePassABuildTileMapKernel, MoePassATileCountsKernel,MoePassATileScanKernel, MoePassAWriteDispatchArgsKernel,@@ -68,37 +68,31 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_counts(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoePassATileCountsArguments<B>,) {- self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, command_buffer);+ self.counts.encode(args.expert_offsets, args.tile_counts, args.e as u32, args.h_blocks, encoder);}pub fn encode_scan(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoePassATileScanArguments<B>,) {- self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, command_buffer);+ self.scan.encode(args.tile_counts, args.tile_offsets, args.total_tiles, args.e as u32, encoder);}pub fn encode_row_map(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoePassARowMapArguments<B>,) {- self.row_map.encode(- args.expert_offsets,- args.row_expert_map,- args.total_rows as u32,- args.e as u32,- command_buffer,- );+ self.row_map.encode(args.expert_offsets, args.row_expert_map, args.total_rows as u32, args.e as u32, encoder);}pub fn encode_build_map(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoePassATileBuildArguments<B>,) {self.build_map.encode(@@ -108,15 +102,15 @@ impl<B: Backend> MoePassATileKernels<B> {args.tile_map,args.total_rows as u32,args.h_blocks,- command_buffer,+ encoder,);}pub fn encode_dispatch_args(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: MoePassATileDispatchArguments<B>,) {- self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, command_buffer);+ self.dispatch.encode(args.total_tiles, args.dispatch_args, args.num_tiles_y, encoder);}}diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex ee92a9b76..4e3220a24 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -3,7 +3,7 @@ use std::collections::HashMap;use crate::{DataType,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,gpu_types::QuantizationMode,kernel::{QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,@@ -161,7 +161,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {pub fn encode(&self,- encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,arguments: QuantizedMatmulArguments<B>,) -> Result<(), QuantizedMatmulError<B>> {if arguments.quantization_type != self.quantization_type {diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rsindex 921b4b5a9..400faa501 100644--- a/crates/uzu/src/backends/common/kernel/sampling.rs+++ b/crates/uzu/src/backends/common/kernel/sampling.rs@@ -9,7 +9,7 @@ use thiserror::Error;use crate::{DataType,backends::common::{- Backend, CommandBuffer, Context, Kernels,+ Backend, Context, Encoder, Kernels,gpu_types::ArgmaxPair,kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel, BitmaskKernel, GumbelKernel, MinPKernel,@@ -152,7 +152,7 @@ impl<B: Backend> SamplingKernel<B> {sampling_method: SamplingMethod,batch_size: usize,vocab_size: usize,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), SamplingError<B>> {if batch_size > self.max_batch_size {return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));@@ -168,7 +168,7 @@ impl<B: Backend> SamplingKernel<B> {logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,- command_buffer,+ encoder,);}@@ -186,7 +186,7 @@ impl<B: Backend> SamplingKernel<B> {batch_size as u32,vocab_size as u32,temperature,- command_buffer,+ encoder,);}@@ -197,7 +197,7 @@ impl<B: Backend> SamplingKernel<B> {batch_size as u32,vocab_size as u32,top_k,- command_buffer,+ encoder,);}@@ -208,7 +208,7 @@ impl<B: Backend> SamplingKernel<B> {batch_size as u32,vocab_size as u32,top_p,- command_buffer,+ encoder,);}@@ -219,7 +219,7 @@ impl<B: Backend> SamplingKernel<B> {batch_size as u32,vocab_size as u32,min_p,- command_buffer,+ encoder,);}@@ -229,7 +229,7 @@ impl<B: Backend> SamplingKernel<B> {logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,- command_buffer,+ encoder,);}@@ -242,7 +242,7 @@ impl<B: Backend> SamplingKernel<B> {sampled_tokens_buffer,batch_size as u32,vocab_size as u32,- command_buffer,+ encoder,);},ArgmaxImplementation::TwoPass {@@ -255,14 +255,14 @@ impl<B: Backend> SamplingKernel<B> {partial_results_buffer.borrow_mut().deref_mut(),batch_size as u32,vocab_size as u32,- command_buffer,+ encoder,);final_kernel.encode(partial_results_buffer.borrow().deref(),sampled_tokens_buffer,batch_size as u32,vocab_size as u32,- command_buffer,+ encoder,);},}diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rsindex cb57d9f33..7d944cf18 100644--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs@@ -1,7 +1,7 @@use crate::{DataType,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},},};@@ -55,7 +55,7 @@ impl<B: Backend> SSDPrefillKernels<B> {pub fn encode(&self,- compute_encoder: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,args: SSDPrefillArguments<B>,mode: SSDPrefillMode,) {@@ -84,7 +84,7 @@ impl<B: Backend> SSDPrefillKernels<B> {state_strides.as_slice(),args.channels as u32,args.head_dim as u32,- compute_encoder,+ encoder,)} else {self.single.encode(@@ -105,7 +105,7 @@ impl<B: Backend> SSDPrefillKernels<B> {state_strides.as_slice(),args.channels as u32,args.head_dim as u32,- compute_encoder,+ encoder,)}} else if mode == SSDPrefillMode::Sequential {@@ -127,7 +127,7 @@ impl<B: Backend> SSDPrefillKernels<B> {state_strides.as_slice(),args.channels as u32,args.head_dim as u32,- compute_encoder,+ encoder,)}}diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rsindex 8557fd41d..d31596649 100644--- a/crates/uzu/src/backends/common/mod.rs+++ b/crates/uzu/src/backends/common/mod.rs@@ -1,19 +1,25 @@mod activation_config;+mod allocator;mod backend;mod buffer;mod command_buffer;mod context;+mod encoder;mod event;pub mod gpu_types;+mod hazard_tracker;pub mod kernel;pub use activation_config::ActivationConfig;+pub use allocator::{Allocation, AllocationPool, AllocationType, Allocator};pub use backend::Backend;-pub use buffer::Buffer;+pub use buffer::{Buffer, BufferGpuAddressRangeExt};pub use command_buffer::{- CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending,+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,+ CommandBufferInitial, CommandBufferPending,};pub use context::Context;+pub use encoder::{Completed, Encoder, Executable, Pending};pub use event::Event;+pub use hazard_tracker::Access;pub use kernel::Kernels;diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rsindex d6a2a5a6e..9d1e5e665 100644--- a/crates/uzu/src/backends/cpu/backend.rs+++ b/crates/uzu/src/backends/cpu/backend.rs@@ -1,4 +1,4 @@-use std::cell::RefCell;+use std::cell::{RefCell, UnsafeCell};use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};use crate::backends::common::Backend;@@ -9,10 +9,11 @@ pub struct Cpu;impl Backend for Cpu {type Context = CpuContext;type CommandBuffer = CpuCommandBuffer;- type Buffer = Box<[u8]>;+ type Buffer = UnsafeCell<Box<[u8]>>;type Event = RefCell<u64>;type Kernels = CpuKernels;type Error = CpuError;+ const MIN_ALLOCATION_ALIGNMENT: usize = 64;const MAX_INLINE_BYTES: usize = usize::MAX;}diff --git a/crates/uzu/src/backends/cpu/buffer.rs b/crates/uzu/src/backends/cpu/buffer.rsindex 306bc05e5..b147cf202 100644--- a/crates/uzu/src/backends/cpu/buffer.rs+++ b/crates/uzu/src/backends/cpu/buffer.rs@@ -1,9 +1,9 @@-use std::{os::raw::c_void, ptr::NonNull};+use std::{cell::UnsafeCell, os::raw::c_void, ptr::NonNull};use super::Cpu;use crate::backends::common::Buffer;-impl Buffer for Box<[u8]> {+impl Buffer for UnsafeCell<Box<[u8]>> {type Backend = Cpu;fn set_label(@@ -13,12 +13,14 @@ impl Buffer for Box<[u8]> {}fn cpu_ptr(&self) -> NonNull<c_void> {- unsafe { NonNull::new_unchecked(self.as_ptr() as *mut c_void) }+ unsafe { NonNull::new_unchecked((&*self.get()).as_ptr() as *mut c_void) }}- fn length(&self) -> usize {- self.len()++ fn gpu_ptr(&self) -> usize {+ unsafe { &*self.get() }.as_ptr().addr()}- fn id(&self) -> usize {- unimplemented!()++ fn length(&self) -> usize {+ unsafe { &*self.get() }.len()}}diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rsindex 1896a56f1..8ce9063bd 100644--- a/crates/uzu/src/backends/cpu/command_buffer.rs+++ b/crates/uzu/src/backends/cpu/command_buffer.rs@@ -1,13 +1,13 @@use std::{- cell::{OnceCell, RefCell},+ cell::{OnceCell, RefCell, UnsafeCell},time::{Duration, Instant},};use super::Cpu;use crate::backends::{common::{- CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending,+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,+ CommandBufferInitial, CommandBufferPending,},cpu::error::CpuError,};@@ -58,12 +58,12 @@ impl CommandBufferEncoding for CpuCommandBuffer {fn encode_copy(&mut self,- src: &Box<[u8]>,- dst: &mut Box<[u8]>,+ src: &UnsafeCell<Box<[u8]>>,+ dst: &mut UnsafeCell<Box<[u8]>>,size: usize,) {- let src = src.as_ptr();- let dst = dst.as_ptr() as *mut u8;+ let src = unsafe { &*src.get() }.as_ptr();+ let dst = dst.get_mut().as_mut_ptr();self.push_command(Box::new(move || unsafe {std::ptr::copy(src, dst, size);}))@@ -71,17 +71,24 @@ impl CommandBufferEncoding for CpuCommandBuffer {fn encode_fill(&mut self,- dst: &mut Box<[u8]>,+ dst: &mut UnsafeCell<Box<[u8]>>,range: std::ops::Range<usize>,value: u8,) {let size = range.end - range.start;- let dst = dst[range].as_ptr() as *mut u8;+ let dst = dst.get_mut()[range].as_ptr() as *mut u8;self.push_command(Box::new(move || unsafe {dst.write_bytes(value, size);}))}+ fn encode_barrier(+ &mut self,+ _after: AccessFlags,+ _before: AccessFlags,+ ) {+ }+fn encode_wait_for_event(&mut self,event: &RefCell<u64>,diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rsindex 43e1c13b2..e0d5e314d 100644--- a/crates/uzu/src/backends/cpu/context.rs+++ b/crates/uzu/src/backends/cpu/context.rs@@ -1,15 +1,23 @@-use std::{cell::RefCell, path::Path, rc::Rc};+use std::{+ cell::{RefCell, UnsafeCell},+ path::Path,+ rc::Rc,+};use super::{Cpu, command_buffer::CpuCommandBuffer, error::CpuError};-use crate::backends::common::Context;+use crate::backends::common::{Allocation, AllocationPool, AllocationType, Allocator, Context};-pub struct CpuContext;+pub struct CpuContext {+ allocator: Rc<Allocator<Cpu>>,+}impl Context for CpuContext {type Backend = Cpu;fn new() -> Result<Rc<Self>, CpuError> {- Ok(Rc::new(CpuContext))+ Ok(Rc::new_cyclic(|weak_self| CpuContext {+ allocator: Allocator::new(weak_self.clone()),+ }))}fn recommended_async_batch_size(@@ -30,8 +38,23 @@ impl Context for CpuContext {fn create_buffer(&self,size: usize,- ) -> Result<Box<[u8]>, CpuError> {- Ok(vec![0; size].into_boxed_slice())+ ) -> Result<UnsafeCell<Box<[u8]>>, CpuError> {+ Ok(UnsafeCell::new(vec![0; size].into_boxed_slice()))+ }++ fn create_allocation(+ &self,+ size: usize,+ allocation_type: AllocationType<Cpu>,+ ) -> Result<Allocation<Cpu>, CpuError> {+ self.allocator.allocate(size, allocation_type)+ }++ fn create_allocation_pool(+ &self,+ reusable: bool,+ ) -> AllocationPool<Cpu> {+ self.allocator.create_pool(reusable)}fn create_command_buffer(&self) -> Result<CpuCommandBuffer, CpuError> {diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 0e9cb49d5..3991c4f8a 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -3,8 +3,11 @@ use half::{bf16, f16};use crate::{DataType,backends::{- common::kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},- cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext},+ common::{+ Encoder,+ kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel},+ },+ cpu::{Cpu, context::CpuContext},},};@@ -31,8 +34,9 @@ impl MatmulKernel for MatmulCpuKernel {&mut self,_context: &CpuContext,arguments: MatmulArguments<Cpu>,- command_buffer: &mut CpuCommandBuffer,+ encoder: &mut Encoder<Cpu>,) {+ let command_buffer = encoder.as_command_buffer_mut();let m = arguments.batch as usize;let n = arguments.output_dim as usize;let k = arguments.input_dim as usize;@@ -42,10 +46,10 @@ impl MatmulKernel for MatmulCpuKernel {let a_offset = arguments.a_offset as usize;let data_type = self.data_type;- let a_ptr = arguments.a.as_ptr().wrapping_byte_add(a_offset);- let b_ptr = arguments.b.as_ptr();- let d_ptr = arguments.d.as_mut_ptr();- let bias_ptr = arguments.bias.map(|b| b.as_ptr());+ let a_ptr = unsafe { &*arguments.a.get() }.as_ptr().wrapping_byte_add(a_offset);+ let b_ptr = unsafe { &*arguments.b.get() }.as_ptr();+ let d_ptr = arguments.d.get_mut().as_mut_ptr();+ let bias_ptr = arguments.bias.map(|b| unsafe { &*b.get() }.as_ptr());command_buffer.push_command(move || match data_type {DataType::F32 => {diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex 97b583bda..bd242efb9 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -15,6 +15,7 @@ impl Backend for Metal {type Kernels = MetalKernels;type Error = MetalError;+ const MIN_ALLOCATION_ALIGNMENT: usize = 64;// Metal's set_bytes supports up to 4KB per bound value.// https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objcconst MAX_INLINE_BYTES: usize = 4096;diff --git a/crates/uzu/src/backends/metal/buffer.rs b/crates/uzu/src/backends/metal/buffer.rsindex 84c0a4914..85d4fe4de 100644--- a/crates/uzu/src/backends/metal/buffer.rs+++ b/crates/uzu/src/backends/metal/buffer.rs@@ -20,11 +20,11 @@ impl Buffer for Retained<ProtocolObject<dyn MTLBuffer>> {self.contents()}- fn length(&self) -> usize {- (**self).length()+ fn gpu_ptr(&self) -> usize {+ self.gpu_address() as usize}- fn id(&self) -> usize {- Retained::as_ptr(self) as usize+ fn length(&self) -> usize {+ (**self).length()}}diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex e27990986..894b5eb96 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -9,8 +9,8 @@ use objc2::{Message, rc::Retained, runtime::ProtocolObject};use super::Metal;use crate::backends::{common::{- CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending,+ AccessFlags, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,+ CommandBufferInitial, CommandBufferPending,},metal::error::MetalError,};@@ -141,6 +141,13 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {self.ensure_blit().fill_buffer_range_value(dst, range, value);}+ fn encode_barrier(+ &mut self,+ _after: AccessFlags,+ _before: AccessFlags,+ ) {+ }+fn encode_wait_for_event(&mut self,event: &Retained<ProtocolObject<dyn MTLEvent>>,diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex de0547a48..30db27a09 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -15,13 +15,17 @@ use super::{metal_extensions::{DeviceGeneration, LibraryPipelineExtensions},};use crate::{- backends::{common::Context, metal::command_buffer::MetalCommandBufferInitial},+ backends::{+ common::{Allocation, AllocationPool, AllocationType, Allocator, Context},+ metal::command_buffer::MetalCommandBufferInitial,+ },utils::ModelSize,};pub struct MetalContext {pub device: Retained<ProtocolObject<dyn MTLDevice>>,pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,+ allocator: Rc<Allocator<Metal>>,device_capabilities: MetalDeviceCapabilities,library: Retained<ProtocolObject<dyn MTLLibrary>>,pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,@@ -66,9 +70,10 @@ impl Context for MetalContext {let device_capabilities = MetalDeviceCapabilities::from_device(&device);- Ok(Rc::new(Self {+ Ok(Rc::new_cyclic(|weak_self| Self {device,command_queue,+ allocator: Allocator::new(weak_self.clone()),device_capabilities,library,pipeline_cache: RefCell::new(HashMap::new()),@@ -107,6 +112,21 @@ impl Context for MetalContext {self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).ok_or(MetalError::CannotCreateBuffer)}+ fn create_allocation(+ &self,+ size: usize,+ allocation_type: AllocationType<Metal>,+ ) -> Result<Allocation<Metal>, MetalError> {+ self.allocator.allocate(size, allocation_type)+ }++ fn create_allocation_pool(+ &self,+ reusable: bool,+ ) -> AllocationPool<Metal> {+ self.allocator.create_pool(reusable)+ }+fn create_command_buffer(&self) -> Result<MetalCommandBufferInitial, MetalError> {Ok(MetalCommandBufferInitial::new(self.command_queue.command_buffer().ok_or(MetalError::CannotCreateCommandBuffer)?,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rsindex f0dada6f4..ae1d4d400 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.rs@@ -11,13 +11,14 @@ use crate::{DataType,backends::{common::{+ Encoder,gpu_types::GemmParams,kernel::{TensorAddBiasKernel,matmul::{MatmulArguments, MatmulError, MatmulKernel},},},- metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext},+ metal::{Metal, context::MetalContext},},};@@ -109,7 +110,7 @@ impl MatmulMetalKernel {fn encode_gemv(&mut self,context: &MetalContext,- command_buffer: &mut MetalCommandBufferEncoding,+ encoder: &mut Encoder<Metal>,arguments: MatmulArguments<Metal>,) -> Result<(), MatmulError<Metal>> {let m = arguments.batch;@@ -182,7 +183,7 @@ impl MatmulMetalKernel {bias_stride,arguments.batch,specialization.output_rows_per_threadgroup() as i32,- command_buffer,+ encoder,);let bias_is_fused = apply_output_scale_and_accumulate && has_bias;@@ -193,7 +194,7 @@ impl MatmulMetalKernel {arguments.d,arguments.batch as usize,arguments.output_dim as usize,- command_buffer,+ encoder,)?;}@@ -203,7 +204,7 @@ impl MatmulMetalKernel {fn encode_gemm(&mut self,context: &MetalContext,- command_buffer: &mut MetalCommandBufferEncoding,+ encoder: &mut Encoder<Metal>,arguments: MatmulArguments<Metal>,) -> Result<(), MatmulError<Metal>> {let m = arguments.batch;@@ -247,7 +248,7 @@ impl MatmulMetalKernel {std::slice::from_ref(¶ms),group_count_x,group_count_y,- command_buffer,+ encoder,);self.encode_bias_add(@@ -256,7 +257,7 @@ impl MatmulMetalKernel {arguments.d,arguments.batch as usize,arguments.output_dim as usize,- command_buffer,+ encoder,)?;Ok(())@@ -269,7 +270,7 @@ impl MatmulMetalKernel {output: &mut <Metal as crate::backends::common::Backend>::Buffer,batch: usize,output_dim: usize,- command_buffer: &mut MetalCommandBufferEncoding,+ encoder: &mut Encoder<Metal>,) -> Result<(), MatmulError<Metal>> {let Some(bias) = bias else {return Ok(());@@ -289,7 +290,7 @@ impl MatmulMetalKernel {output,output_dim as u32,total_length as u32,- command_buffer,+ encoder,);Ok(())}@@ -327,12 +328,12 @@ impl MatmulKernel for MatmulMetalKernel {&mut self,context: &MetalContext,arguments: MatmulArguments<Metal>,- command_buffer: &mut MetalCommandBufferEncoding,+ encoder: &mut Encoder<Metal>,) {if Self::is_gemv_eligible(&arguments) {- self.encode_gemv(context, command_buffer, arguments).expect("Failed to encode GEMV kernel");+ self.encode_gemv(context, encoder, arguments).expect("Failed to encode GEMV kernel");} else {- self.encode_gemm(context, command_buffer, arguments).expect("Failed to encode GEMM kernel");+ self.encode_gemm(context, encoder, arguments).expect("Failed to encode GEMM kernel");}}}diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex 93badc6a7..181d9afa0 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -152,4 +152,4 @@ PUBLIC KERNEL(MoeRouterTopK)(out_probs[i] = static_cast<ScalarT>(0.0f);}}-}\ No newline at end of file+}diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 1b16fe499..c43a205aa 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -7,9 +7,7 @@ use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{DataType,- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- },+ backends::common::{Backend, Encoder},encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,@@ -108,54 +106,32 @@ impl<B: Backend> Classifier<B> {let encoding_params = EncodingParameters::new();- let mut command_buffer = self- .context- .context- .create_command_buffer()- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?- .start_encoding();-- self.context- .embed- .encode_lookup(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .embedding_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;++ self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());+ state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().embedding_norm().clone());}for layer in self.context.layers.iter() {- layer- .encode(&mut state, &encoding_params, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;}- self.context- .output_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());+ state.encode_copy_array(&mut encoder, ArrayId::Main, traces.borrow().output_norm.clone());}- self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .prediction_head- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- command_buffer- .end_encoding()- .submit()- .wait_until_completed()- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;let logits = self.copy_logits_from_state(&state)?;@@ -183,41 +159,19 @@ impl<B: Backend> Classifier<B> {let encoding_params = EncodingParameters::new();- let mut command_buffer = self- .context- .context- .create_command_buffer()- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?- .start_encoding();-- self.context- .embed- .encode_lookup(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .embedding_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;++ self.context.embed.encode_lookup(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.embedding_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;for layer in self.context.layers.iter() {- layer- .encode(&mut state, &encoding_params, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ layer.encode(&mut state, &encoding_params, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;}- self.context- .output_norm- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context.pooling.encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .prediction_head- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- command_buffer- .end_encoding()- .submit()- .wait_until_completed()- .map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;+ self.context.output_norm.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.pooling.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.prediction_head.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;++ encoder.end_encoding().submit().wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;let logits = self.copy_logits_from_state(&state)?;diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rsindex 810b64344..0bc689869 100644--- a/crates/uzu/src/encodable_block/activation.rs+++ b/crates/uzu/src/encodable_block/activation.rs@@ -5,7 +5,7 @@ use std::ops::DerefMut;use crate::{DataType,backends::common::{- ActivationConfig, Backend, CommandBuffer,+ ActivationConfig, Backend, Encoder,gpu_types::ActivationType,kernel::{ActivationKernel, Kernels},},@@ -40,7 +40,7 @@ impl<B: Backend> Activation<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let input_array = arrays[0].borrow();@@ -58,7 +58,7 @@ impl<B: Backend> Activation<B> {output_array.buffer().borrow_mut().deref_mut(),n as u32,self.activation.act_type(),- command_buffer,+ encoder,);Ok(())}diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex c5dad382e..a66bbf4da 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -8,7 +8,7 @@ use std::{use crate::{DataType,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,SigmoidGateKernel,@@ -147,7 +147,7 @@ impl<B: Backend> Attention<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);@@ -258,7 +258,7 @@ impl<B: Backend> Attention<B> {suffix_length as u32,0u32,max_sequence_length as u32,- command_buffer,+ encoder,);}@@ -318,7 +318,7 @@ impl<B: Backend> Attention<B> {suffix_length as u32,segment_prefix_length as u32,max_sequence_length as u32,- command_buffer,+ encoder,);}@@ -371,9 +371,7 @@ impl<B: Backend> Attention<B> {is_causal: self.is_causal,scale,};- self.gemm_block- .encode(state.context(), command_buffer, args)- .expect("Failed to encode AttentionGemmBlock");+ self.gemm_block.encode(state.context(), encoder, args).expect("Failed to encode AttentionGemmBlock");},KernelVariant::SinglePass => {let kernel = match self.single_pass_kernels.get(&kernel_key) {@@ -399,7 +397,7 @@ impl<B: Backend> Attention<B> {sinks_buffer,num_heads as u32,suffix_length as u32,- command_buffer,+ encoder,)},KernelVariant::TwoPass => {@@ -432,7 +430,7 @@ impl<B: Backend> Attention<B> {mask_q_seq_stride_opt,mask_head_stride_opt,sinks_buffer,- command_buffer,+ encoder,);kernel_pass2.encode(partials_buf_borrow.deref(),@@ -441,7 +439,7 @@ impl<B: Backend> Attention<B> {attention_output_buf_borrow.deref_mut(),num_heads as u32,suffix_length as u32,- command_buffer,+ encoder,);},}@@ -455,7 +453,7 @@ impl<B: Backend> Attention<B> {gate_buf_borrow.deref(),attention_output_buf_borrow.deref_mut(),total_elements,- command_buffer,+ encoder,);}diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex 5ead1cf31..a2344e829 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -2,7 +2,7 @@ use std::rc::Rc;use crate::{DataType,- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, Encoder},config::TransformerLayerConfig,encodable_block::{Attention, EncodingParameters, Linear, Mlp, Normalization, QKNorm, Rope, TensorAddSwap, TensorCopy,@@ -225,95 +225,83 @@ impl<B: Backend> ClassifierLayer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {#[cfg(feature = "tracing")]let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());}- self.copy_main_to_shortcut_mixer.encode(state, command_buffer)?;+ self.copy_main_to_shortcut_mixer.encode(state, encoder)?;if let Some(ref pre_attn_norm) = self.pre_attention_norm {- pre_attn_norm.encode(state, command_buffer)?;+ pre_attn_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().pre_attention_norm.clone(),- );+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());}} else {#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().pre_attention_norm.clone(),- );+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());}}- self.qkv_projection.encode(state, command_buffer)?;+ self.qkv_projection.encode(state, encoder)?;if let Some(ref qk_norm) = self.qk_norm {- qk_norm.encode(state, command_buffer)?;+ qk_norm.encode(state, encoder)?;}- self.rope.encode(state, command_buffer)?;- self.attention.encode(state, parameters, command_buffer)?;- self.out_projection.encode(state, command_buffer)?;+ self.rope.encode(state, encoder)?;+ self.attention.encode(state, parameters, encoder)?;+ self.out_projection.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());}if let Some(ref post_attn_norm) = self.post_attention_norm {- post_attn_norm.encode(state, command_buffer)?;+ post_attn_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().post_attention_norm.clone(),- );+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());}}- self.mixer_residual_add.encode(state, command_buffer)?;+ self.mixer_residual_add.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());}- self.copy_main_to_shortcut_mlp.encode(state, command_buffer)?;+ self.copy_main_to_shortcut_mlp.encode(state, encoder)?;- self.pre_mlp_norm.encode(state, command_buffer)?;+ self.pre_mlp_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());}- self.mlp.encode(state, command_buffer)?;+ self.mlp.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());}if let Some(ref post_mlp_norm) = self.post_mlp_norm {- post_mlp_norm.encode(state, command_buffer)?;+ post_mlp_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());}}- self.mlp_residual_add.encode(state, command_buffer)?;+ self.mlp_residual_add.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());}Ok(())diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex c60e342b2..901a67b12 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -6,7 +6,7 @@ use thiserror::Error;use crate::{DataType, DecoderConfig,- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, Encoder},config::{DecoderLayerType, MixerConfig},encodable_block::{Embedding, EncodingParameters, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError},forward_pass::{@@ -166,30 +166,30 @@ impl<B: Backend> Decoder<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), DecoderError<B>> {- self.embed.encode_lookup(state, command_buffer)?;+ self.embed.encode_lookup(state, encoder)?;for layer in self.layers.iter() {- layer.encode(state, parameters, command_buffer).map_err(DecoderError::BackendError)?;+ layer.encode(state, parameters, encoder).map_err(DecoderError::BackendError)?;}if state.is_prefilling() {return Ok(());}- self.norm.encode(state, command_buffer).map_err(DecoderError::BackendError)?;+ self.norm.encode(state, encoder).map_err(DecoderError::BackendError)?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());+ state.encode_copy_array(encoder, ArrayId::Main, traces.borrow().output_norm.clone());}- self.embed.encode_readout(state, command_buffer)?;+ self.embed.encode_readout(state, encoder)?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(command_buffer, ArrayId::Logits, traces.borrow().logits.clone());+ state.encode_copy_array(encoder, ArrayId::Logits, traces.borrow().logits.clone());}Ok(())}diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex 775dea92f..f58fabf51 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -8,7 +8,7 @@ use thiserror::Error;use crate::{DataType, EmbeddingConfig,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,kernel::{FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},@@ -421,7 +421,7 @@ impl<B: Backend> Embedding<B> {pub fn encode_lookup(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), EmbeddingError<B>> {let batch_size = state.active_suffix_length() as u32;@@ -461,7 +461,7 @@ impl<B: Backend> Embedding<B> {self.vocab_size,self.model_dim,self.input_scale,- command_buffer,+ encoder,),EmbeddingTying::Tied {ty:@@ -493,7 +493,7 @@ impl<B: Backend> Embedding<B> {self.vocab_size,self.model_dim,self.input_scale,- command_buffer,+ encoder,);},};@@ -504,7 +504,7 @@ impl<B: Backend> Embedding<B> {pub fn encode_readout(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), EmbeddingError<B>> {let batch_size = state.sampling_length();@@ -561,7 +561,7 @@ impl<B: Backend> Embedding<B> {leading_dimension_d: output_dim as i32,transpose_b: true,},- command_buffer,+ encoder,);},EmbeddingTying::Tied {@@ -585,7 +585,7 @@ impl<B: Backend> Embedding<B> {},} => {readout.encode(- command_buffer,+ encoder,QuantizedMatmulArguments {a_buffer: input,a_offset: input_offset,diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex f785afb80..d0a55a4d9 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -5,7 +5,7 @@ use std::rc::Rc;use super::MixerExecutables;use crate::{DataType, DecoderLayerConfig,- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, Encoder},config::{DecoderLayerType, MixerConfig},encodable_block::{Attention, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer, TensorAddSwap,@@ -262,23 +262,23 @@ impl<B: Backend> LayerExecutables<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {#[cfg(feature = "tracing")]let layer_traces = state.traces().borrow().layer_results.get(self.layer_index).cloned();#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().inputs.clone());}- self.copy_main_to_shortcut.encode(state, command_buffer)?;+ self.copy_main_to_shortcut.encode(state, encoder)?;// shortcut = input- self.pre_attention_norm.encode(state, command_buffer)?;+ self.pre_attention_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());}match &self.mixer {@@ -290,89 +290,85 @@ impl<B: Backend> LayerExecutables<B> {attention,out_projection,} => {- qkv_projection.encode(state, command_buffer)?;+ qkv_projection.encode(state, encoder)?;if let Some(gate_proj) = gate_projection {- gate_proj.encode(state, command_buffer)?;+ gate_proj.encode(state, encoder)?;}if let Some(norm) = qk_norm {- norm.encode(state, command_buffer)?;+ norm.encode(state, encoder)?;}- rope.encode(state, command_buffer)?;- attention.encode(state, parameters, command_buffer)?;- out_projection.encode(state, command_buffer)?;+ rope.encode(state, encoder)?;+ attention.encode(state, parameters, encoder)?;+ out_projection.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());}},MixerExecutables::StateSpace {mixer,} => {- mixer.encode(state, command_buffer)?;+ mixer.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());}},MixerExecutables::ShortConv {mixer,} => {- mixer.encode(state, command_buffer)?;+ mixer.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().attention.clone());}},}if let Some(post_attention_norm) = &self.post_attention_norm {- post_attention_norm.encode(state, command_buffer)?;+ post_attention_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(- command_buffer,- ArrayId::Main,- layer_traces.borrow().post_attention_norm.clone(),- );+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_attention_norm.clone());}}//main = attention_result- self.main_shortcut_add_swap.encode(state, command_buffer)?;+ self.main_shortcut_add_swap.encode(state, encoder)?;// shortcut = input + attention_result// main = input + attention_result#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());}- self.pre_mlp_norm.encode(state, command_buffer)?;+ self.pre_mlp_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());}- self.mlp.encode(state, command_buffer)?;+ self.mlp.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().mlp.clone());}if let Some(post_mlp_norm) = &self.post_mlp_norm {- post_mlp_norm.encode(state, command_buffer)?;+ post_mlp_norm.encode(state, encoder)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());}}// main = mlp_result- self.main_shortcut_add_swap.encode(state, command_buffer)?;+ self.main_shortcut_add_swap.encode(state, encoder)?;// shortcut = input + attention_result + mlp_result// main = input + attention_result + mlp_result#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {- state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());+ state.encode_copy_array(encoder, ArrayId::Main, layer_traces.borrow().outputs.clone());}Ok(())diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rsindex 301baf9e6..86ed51fe0 100644--- a/crates/uzu/src/encodable_block/layer_norm.rs+++ b/crates/uzu/src/encodable_block/layer_norm.rs@@ -11,7 +11,7 @@ use thiserror::Error;use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, LayerNormKernel},},config::{NormalizationConfig, UpcastMode},@@ -71,7 +71,7 @@ impl<B: Backend> LayerNorm<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);@@ -103,7 +103,7 @@ impl<B: Backend> LayerNorm<B> {self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),full_layer,- command_buffer,+ encoder,);Ok(())}diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rsindex e54049ec7..3e4e59cbf 100644--- a/crates/uzu/src/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs@@ -10,7 +10,7 @@ use super::Linear;use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},},forward_pass::state::{ArrayId, ForwardPassState},@@ -129,7 +129,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();@@ -153,7 +153,7 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {leading_dimension_d: self.output_dim as i32,transpose_b: true,},- command_buffer,+ encoder,);Ok(())}diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rsindex faaf316ab..7d8478fc6 100644--- a/crates/uzu/src/encodable_block/linear/mod.rs+++ b/crates/uzu/src/encodable_block/linear/mod.rs@@ -6,7 +6,7 @@ pub use quantized::{QuantizedLinear, QuantizedLinearError};use thiserror::Error;use crate::{- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, Encoder},config::LinearConfig,forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -16,7 +16,7 @@ pub trait Linear<B: Backend> {fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error>;}diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rsindex 3f46ab886..ebcfcd6a7 100644--- a/crates/uzu/src/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/encodable_block/linear/quantized.rs@@ -10,7 +10,7 @@ use super::Linear;use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, TensorAddBiasKernel,quant_matmul::{@@ -249,7 +249,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();@@ -260,7 +260,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {self.kernel.encode(- command_buffer,+ encoder,QuantizedMatmulArguments {a_buffer: input_array.buffer().borrow().deref(),a_offset: 0,@@ -284,7 +284,7 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {output_buf_borrow.deref_mut(),self.output_dim as u32,total_length as u32,- command_buffer,+ encoder,);}Ok(())diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex bc0d2ade9..035393bb8 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -9,7 +9,7 @@ use crate::{DataType,array::Array,backends::common::{- Backend, CommandBuffer, Kernels,+ Backend, Encoder, Kernels,kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},@@ -146,7 +146,7 @@ impl<B: Backend> MambaMixer<B> {fn run_split_inproj(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,suffix_length: usize,) {let arrays = state.arrays(&[@@ -179,14 +179,14 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,inner_dim as u32,num_heads as u32,- command_buffer,+ encoder,)}fn run_conv_scan(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,suffix_length: usize,) {let arrays = state.arrays(&[@@ -231,7 +231,7 @@ impl<B: Backend> MambaMixer<B> {inner_dim as u32,proj_dim as u32,self.config.activation.act_type(),- command_buffer,+ encoder,);}} else {@@ -247,7 +247,7 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,suffix_length as u32,conv_dim as u32,- command_buffer,+ encoder,);Some(buffer)@@ -276,7 +276,7 @@ impl<B: Backend> MambaMixer<B> {inner_dim as u32,proj_dim as u32,self.config.activation.act_type(),- command_buffer,+ encoder,)}}@@ -285,7 +285,7 @@ impl<B: Backend> MambaMixer<B> {fn run_prefill_ssm(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,suffix_length: usize,) {let base_arrays = state.arrays(&[@@ -309,7 +309,7 @@ impl<B: Backend> MambaMixer<B> {let skip_rc = self.skip_connection_weight.buffer();let skip_borrow = skip_rc.borrow();self.ssd_prefill.encode(- command_buffer,+ encoder,SSDPrefillArguments {x: x.buffer().borrow().deref(),dt: dt.buffer().borrow().deref(),@@ -336,7 +336,7 @@ impl<B: Backend> MambaMixer<B> {fn run_decode_ssm(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,suffix_length: usize,) {let arrays = state.arrays(&[@@ -390,29 +390,29 @@ impl<B: Backend> MambaMixer<B> {suffix_length as u32,h,dh,- command_buffer,+ encoder,);}pub(crate) fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {return Ok(());}- self.in_projection.encode(state, command_buffer)?;- self.run_split_inproj(state, command_buffer, active_suffix_length);- self.run_conv_scan(state, command_buffer, active_suffix_length);+ self.in_projection.encode(state, encoder)?;+ self.run_split_inproj(state, encoder, active_suffix_length);+ self.run_conv_scan(state, encoder, active_suffix_length);if active_suffix_length == 1 {- self.run_decode_ssm(state, command_buffer, active_suffix_length);+ self.run_decode_ssm(state, encoder, active_suffix_length);} else {- self.run_prefill_ssm(state, command_buffer, active_suffix_length);+ self.run_prefill_ssm(state, encoder, active_suffix_length);}- self.out_projection.encode(state, command_buffer)?;+ self.out_projection.encode(state, encoder)?;Ok(())}}diff --git a/crates/uzu/src/encodable_block/mlp/dense.rs b/crates/uzu/src/encodable_block/mlp/dense.rsindex 1c2e9eac1..eaed083d8 100644--- a/crates/uzu/src/encodable_block/mlp/dense.rs+++ b/crates/uzu/src/encodable_block/mlp/dense.rs@@ -4,7 +4,7 @@ use std::ops::{Deref, DerefMut};use super::{super::linear::Linear, Mlp};use crate::{- backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},+ backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},forward_pass::state::{ArrayId, ForwardPassState},};@@ -32,10 +32,10 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {// Up- self.up.encode(state, command_buffer)?;+ self.up.encode(state, encoder)?;// Gate act+mul (fused_up -> hidden)let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);@@ -47,7 +47,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {let hidden_buf_rc = hidden.buffer();let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();self.gate- .encode(command_buffer, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)+ .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m).expect("Failed to encode MLP activation/mul kernel");drop(hidden_buf_borrow);drop(fused_buf_borrow);@@ -55,7 +55,7 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {drop(hidden);// Down- self.down.encode(state, command_buffer)?;+ self.down.encode(state, encoder)?;Ok(())}}diff --git a/crates/uzu/src/encodable_block/mlp/mod.rs b/crates/uzu/src/encodable_block/mlp/mod.rsindex e6d28a089..23cb57405 100644--- a/crates/uzu/src/encodable_block/mlp/mod.rs+++ b/crates/uzu/src/encodable_block/mlp/mod.rs@@ -8,7 +8,7 @@ use thiserror::Error;use super::linear::{Linear, LinearBlockError};use crate::{DataType,- backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},+ backends::common::{Backend, Encoder, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},config::MLPConfig,forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},@@ -18,7 +18,7 @@ pub trait Mlp<B: Backend> {fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error>;}diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rsindex 4339670ef..62b2d1bb1 100644--- a/crates/uzu/src/encodable_block/mlp/moe.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -12,7 +12,7 @@ use crate::{DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,array::Array,backends::common::{- Backend, CommandBuffer, CommandBufferEncoding, Kernels,+ Backend, Encoder, Kernels,gpu_types::ActivationType,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,@@ -193,7 +193,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let suffix_length = state.active_suffix_length();let arrays = state.arrays(&[@@ -256,28 +256,28 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {// Clear topk_ids and tok2row buffersif topk_bytes > 0 {- command_buffer.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);+ encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);}if tok2row_bytes > 0 {- command_buffer.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);+ encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);}// Clear hidden bufferlet hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();if hidden_bytes > 0 {- command_buffer.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);+ encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);}// Clear y_partial bufferlet y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();if y_partial_bytes > 0 {- command_buffer.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);+ encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);}// Clear x_perm bufferlet x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();if x_perm_bytes > 0 {- command_buffer.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);+ encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);}}@@ -305,7 +305,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {e as u32,k as u32,self.router_renorm,- command_buffer,+ encoder,);}@@ -317,7 +317,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {suffix_length as u32,e as u32,k as u32,- command_buffer,+ encoder,);let num_blocks = ((suffix_length + 255) / 256).max(1);@@ -330,7 +330,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {num_blocks as u32,num_tiles as u32,0u32,- command_buffer,+ encoder,);self.scatter_map_kernel.encode(topk_ids_buf.borrow().deref(),@@ -346,11 +346,11 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {num_blocks as u32,num_tiles as u32,tok2row_buf.borrow_mut().deref_mut(),- command_buffer,+ encoder,);self.gather_kernels.encode(- command_buffer,+ encoder,self.data_type,MoeGatherArguments {x_buffer: main_buf.borrow().deref(),@@ -377,7 +377,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;self.experts_two_pass_decode_kernel.encode(- command_buffer,+ encoder,MoeExpertsTwoPassArguments {x_perm_buffer: x_perm_buf.borrow().deref(),expert_offsets: offsets_buf.borrow().deref(),@@ -454,7 +454,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {silu_alpha,data_type: self.data_type,};- self.experts_two_pass_prefill_kernel.encode(command_buffer, args);+ self.experts_two_pass_prefill_kernel.encode(encoder, args);}self.finalize_kernel.encode(@@ -465,7 +465,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {suffix_length as u32,self.model_dim as u32,k as u32,- command_buffer,+ encoder,);Ok(())diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rsindex 53d5cfd51..7c99d0873 100644--- a/crates/uzu/src/encodable_block/normalization.rs+++ b/crates/uzu/src/encodable_block/normalization.rs@@ -5,7 +5,7 @@ use thiserror::Error;use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};use crate::{DataType,- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, Encoder},config::NormalizationConfig,forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -60,11 +60,11 @@ impl<B: Backend> Normalization<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {match self {- Self::LayerNorm(layer_norm) => layer_norm.encode(state, command_buffer),- Self::RMSNorm(rms_norm) => rms_norm.encode(state, command_buffer),+ Self::LayerNorm(layer_norm) => layer_norm.encode(state, encoder),+ Self::RMSNorm(rms_norm) => rms_norm.encode(state, encoder),}}}diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rsindex b6cfba724..74903ab54 100644--- a/crates/uzu/src/encodable_block/pooling.rs+++ b/crates/uzu/src/encodable_block/pooling.rs@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, PoolingClsKernel, PoolingMeanKernel},},config::PoolingType,@@ -25,11 +25,11 @@ impl<B: Backend> PoolingKernel<B> {seq_len: u32,hidden_dim: u32,batch_size: u32,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) {match self {- Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),- Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, command_buffer),+ Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),+ Self::Mean(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),}}}@@ -61,7 +61,7 @@ impl<B: Backend> Pooling<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let batch_size = 1;let seq_len = state.aux_buffers_suffix_length();@@ -76,13 +76,13 @@ impl<B: Backend> Pooling<B> {seq_len as u32,self.model_dim as u32,batch_size,- command_buffer,+ encoder,);#[cfg(feature = "tracing")]{let output_pooling_trace = state.traces().borrow().output_pooling().clone();- state.encode_copy_array(command_buffer, ArrayId::ClassifierPooling, output_pooling_trace);+ state.encode_copy_array(encoder, ArrayId::ClassifierPooling, output_pooling_trace);}Ok(())}diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rsindex 62e62ba44..aca2827c7 100644--- a/crates/uzu/src/encodable_block/prediction_head.rs+++ b/crates/uzu/src/encodable_block/prediction_head.rs@@ -3,7 +3,7 @@#[cfg(feature = "tracing")]use crate::forward_pass::state::ArrayId;use crate::{- backends::common::{Backend, CommandBuffer},+ backends::common::{Backend, Encoder},encodable_block::{Activation, Normalization, linear::Linear},forward_pass::state::ForwardPassState,};@@ -37,21 +37,17 @@ impl<B: Backend> ClassifierPredictionHead<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- self.dense.encode(state, command_buffer)?;- self.activation.encode(state, command_buffer)?;- self.norm.encode(state, command_buffer)?;- self.readout.encode(state, command_buffer)?;+ self.dense.encode(state, encoder)?;+ self.activation.encode(state, encoder)?;+ self.norm.encode(state, encoder)?;+ self.readout.encode(state, encoder)?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(- command_buffer,- ArrayId::ClassifierPredictionHeadLogits,- traces.borrow().logits.clone(),- );+ state.encode_copy_array(encoder, ArrayId::ClassifierPredictionHeadLogits, traces.borrow().logits.clone());}Ok(())diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rsindex 6c5a7cdab..9ec1ebf44 100644--- a/crates/uzu/src/encodable_block/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -11,7 +11,7 @@ use thiserror::Error;use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, QKNormKernel},},config::{NormalizationConfig, UpcastMode},@@ -124,7 +124,7 @@ impl<B: Backend> QKNorm<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let qkv_binding = state.arrays(&[self.qkv_array_id]);let qkv_shape = {@@ -152,7 +152,7 @@ impl<B: Backend> QKNorm<B> {0,self.num_q_heads as u32,query_config.upcast_mode == UpcastMode::FullLayer,- command_buffer,+ encoder,);}@@ -173,7 +173,7 @@ impl<B: Backend> QKNorm<B> {self.num_q_heads as u32,self.num_kv_heads as u32,key_config.upcast_mode == UpcastMode::FullLayer,- command_buffer,+ encoder,);}Ok(())diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 30ecb88f1..3745391e4 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -11,7 +11,7 @@ use thiserror::Error;use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, RMSNormKernel},},config::{NormalizationConfig, UpcastMode},@@ -85,7 +85,7 @@ impl<B: Backend> RMSNorm<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);@@ -130,7 +130,7 @@ impl<B: Backend> RMSNorm<B> {self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),self.config.upcast_mode == UpcastMode::FullLayer,- command_buffer,+ encoder,);Ok(())}diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rsindex edee08bcd..8b23c8e99 100644--- a/crates/uzu/src/encodable_block/rope.rs+++ b/crates/uzu/src/encodable_block/rope.rs@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, RopeKernel},},forward_pass::state::{ArrayId, ForwardPassState, RopeType},@@ -31,7 +31,7 @@ impl<B: Backend> Rope<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let (suffix_length, num_heads, head_dim, num_groups, rope_dim, rope_max_seq_len) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);@@ -89,7 +89,7 @@ impl<B: Backend> Rope<B> {num_groups as u32,suffix_length as u32,rope_max_seq_len as u32,- command_buffer,+ encoder,);Ok(())}diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex 8b174add0..74c15aa24 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::sampling::{SamplingError, SamplingKernel},},forward_pass::state::{ArrayId, ForwardPassState},@@ -31,7 +31,7 @@ impl<B: Backend> Sampling<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");@@ -77,7 +77,7 @@ impl<B: Backend> Sampling<B> {sampling_method,batch_size,vocab_size,- command_buffer,+ encoder,) {panic!("Sampling encoding failed: {:?}", e);}diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex 98406a2e7..d98a6b322 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -4,7 +4,7 @@ use crate::{DataType,array::Array,backends::common::{- Backend, CommandBuffer, Context, Kernels,+ Backend, Context, Encoder, Kernels,kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},},config::{DecoderLayerType, ShortConvConfig},@@ -130,19 +130,19 @@ impl<B: Backend> ShortConvMixer<B> {fn run_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,active_suffix_length: usize,) {self.clear_suffix_state_valid_range(state);if active_suffix_length == 1 {- self.run_decode_conv(state, compute, 1);+ self.run_decode_conv(state, encoder, 1);return;}let sampling_len = state.sampling_length();if sampling_len == 0 {- self.run_prefill_conv(state, compute, active_suffix_length);+ self.run_prefill_conv(state, encoder, active_suffix_length);return;}@@ -150,26 +150,26 @@ impl<B: Backend> ShortConvMixer<B> {let trie_len = sampling_len;if trie_len <= 1 {- self.run_prefill_conv(state, compute, active_suffix_length);+ self.run_prefill_conv(state, encoder, active_suffix_length);return;}if sampling_start > 0 {if sampling_start == 1 {- self.run_decode_conv(state, compute, 1);+ self.run_decode_conv(state, encoder, 1);} else {- self.run_prefill_conv(state, compute, sampling_start);+ self.run_prefill_conv(state, encoder, sampling_start);}}- self.run_trie_conv(state, compute, sampling_start, trie_len);+ self.run_trie_conv(state, encoder, sampling_start, trie_len);self.set_suffix_state_valid_range(state, sampling_start, trie_len);}fn run_prefill_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -204,7 +204,7 @@ impl<B: Backend> ShortConvMixer<B> {suffix_length as u32,self.model_dim as u32 * 3,self.model_dim as u32,- compute,+ encoder,);self.short_conv_prefill.encode(@@ -219,14 +219,14 @@ impl<B: Backend> ShortConvMixer<B> {self.model_dim as u32 * 3,state_stride as u32,self.model_dim as u32,- compute,+ encoder,)}fn run_trie_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,sampling_start: usize,trie_len: usize,) {@@ -278,14 +278,14 @@ impl<B: Backend> ShortConvMixer<B> {in_proj_stride as u32,state_stride as u32,self.model_dim as u32,- compute,+ encoder,);}fn run_decode_conv(&self,state: &mut ForwardPassState<B>,- compute: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -317,25 +317,25 @@ impl<B: Backend> ShortConvMixer<B> {self.model_dim as u32 * 3,state_stride as u32,self.model_dim as u32,- compute,+ encoder,)}pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {return Ok(());}- self.in_projection.encode(state, command_buffer)?;+ self.in_projection.encode(state, encoder)?;- self.run_conv(state, command_buffer, active_suffix_length);+ self.run_conv(state, encoder, active_suffix_length);- self.out_projection.encode(state, command_buffer)?;+ self.out_projection.encode(state, encoder)?;Ok(())}diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rsindex ea0b2b9d1..05db52008 100644--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs@@ -5,7 +5,7 @@ use std::ops::DerefMut;use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, TensorAddSwapKernel},},forward_pass::state::{ArrayId, ForwardPassState},@@ -32,7 +32,7 @@ impl<B: Backend> TensorAddSwap<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");@@ -46,7 +46,7 @@ impl<B: Backend> TensorAddSwap<B> {skip_array.buffer().borrow_mut().deref_mut(),main_array.buffer().borrow_mut().deref_mut(),length as u32,- command_buffer,+ encoder,);Ok(())}diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rsindex 74af8ac92..00f0ec5af 100644--- a/crates/uzu/src/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/encodable_block/tensor_copy.rs@@ -5,7 +5,7 @@ use std::ops::{Deref, DerefMut};use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::{Kernels, TensorCopyKernel},},forward_pass::state::{ArrayId, ForwardPassState},@@ -32,7 +32,7 @@ impl<B: Backend> TensorCopy<B> {pub fn encode(&self,state: &mut ForwardPassState<B>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");@@ -46,7 +46,7 @@ impl<B: Backend> TensorCopy<B> {source_array.buffer().borrow().deref(),destination_array.buffer().borrow_mut().deref_mut(),length as u32,- command_buffer,+ encoder,);Ok(())}diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rsindex 8a88a6741..f80eeadee 100644--- a/crates/uzu/src/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -5,7 +5,7 @@ use std::{use crate::{array::{Array, ArrayContextExt},- backends::common::{Backend, CommandBuffer, kernel::kv_cache_update::KVCacheUpdate},+ backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},config::DecoderLayerType,forward_pass::{kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState, KVSlice},@@ -276,13 +276,13 @@ impl<B: Backend> CacheLayers<B> {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,kv_cache_update: &KVCacheUpdate<B>,) {let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);for layer in self.data.iter_mut() {if let Some(layer) = layer.as_transformer_mut() {- layer.update_after_acceptance(accepted_suffix_indices, suffix_start, command_buffer, kv_cache_update);+ layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);} else if let Some(layer) = layer.as_short_conv() {layer.commit_from_suffix_state_if_valid(short_conv_commit_index);}diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rsindex b60364064..6b0ffbe45 100644--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs@@ -1,7 +1,7 @@use crate::{array::{Array, ArrayCell, ArrayContextExt},backends::common::{- Backend, CommandBuffer,+ Backend, Encoder,kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},},utils::attention::fill_attention_bias,@@ -163,7 +163,7 @@ impl<B: Backend> KVCacheLayer<B> {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,kv_cache_update: &KVCacheUpdate<B>,) {match &mut self.state {@@ -182,7 +182,7 @@ impl<B: Backend> KVCacheLayer<B> {let destination_indices: Vec<usize> =(*prefix_len..*prefix_len + accepted_suffix_indices.len()).collect();- self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);+ self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);},KVCacheLayerState::Windowed {@@ -204,7 +204,7 @@ impl<B: Backend> KVCacheLayer<B> {destination_indices.push((*ring_length + *ring_offset + i) % *window_length);}- self.scatter_if_required(&source_indices, &destination_indices, command_buffer, kv_cache_update);+ self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);},}}@@ -213,7 +213,7 @@ impl<B: Backend> KVCacheLayer<B> {&self,source_indices: &[usize],destination_indices: &[usize],- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,kv_cache_update: &KVCacheUpdate<B>,) {if source_indices == destination_indices {@@ -230,7 +230,7 @@ impl<B: Backend> KVCacheLayer<B> {value_shape: [v_shape[0], v_shape[1], v_shape[2]],};- let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, command_buffer);+ let _ = kv_cache_update.encode(&[layer_data], source_indices, destination_indices, encoder);}pub fn register_accepted_tokens(diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex 8112db011..50e1c2ca2 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -3,7 +3,7 @@ use std::ops::{Deref, DerefMut};use std::{cell::RefCell, collections::HashMap, rc::Rc};#[cfg(feature = "tracing")]-use crate::backends::common::{CommandBuffer, CommandBufferEncoding};+use crate::backends::common::Encoder;#[cfg(feature = "tracing")]use crate::forward_pass::traces::ActivationTrace;use crate::{@@ -751,7 +751,7 @@ impl<B: Backend> ForwardPassState<B> {#[cfg(feature = "tracing")]pub fn encode_copy_array(&self,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ encoder: &mut Encoder<B>,source_array_id: ArrayId,destination_array: RefCell<Array<B>>,) {@@ -765,6 +765,6 @@ impl<B: Backend> ForwardPassState<B> {let copy_size_bytes = dst_borrow.size();debug_assert_eq!(dst_borrow.size(), src_borrow.size());- command_buffer.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);+ encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);}}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex ac08f8599..7245ecfda 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -17,8 +17,7 @@ use super::{};use crate::{backends::common::{- Backend, Buffer, CommandBuffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending, Context,+ Backend, Buffer, CommandBuffer, Context, Encoder, Executable,kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},},encodable_block::EncodingParameters,@@ -67,7 +66,7 @@ pub struct LanguageModelGenerator<B: Backend> {pub tokens: Vec<u64>,pub context: LanguageModelGeneratorContext<B>,- pre_encoded_task: Option<(TaskEncodingKey, <B::CommandBuffer as CommandBuffer>::Executable)>,+ pre_encoded_task: Option<(TaskEncodingKey, Executable<B>)>,registered_prefix_len: usize,gpu_capture: GpuCaptureManager<B>,}@@ -509,28 +508,21 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {*sm = Some(sampling_method);}- let mut command_buffer = self- .context- .context- .create_command_buffer()- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?- .start_encoding();+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;// Wait on previous pass if this is a continuationif is_continuation {- command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);+ encoder.encode_wait_for_event(&self.context.async_buffers.event, current_counter);}self.context.executables- .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)+ .encode(&mut state, &EncodingParameters::new(), &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;// Encode sampling- self.context- .gpu_sampler- .encode(&mut state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.gpu_sampler.encode(&mut state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;// Copy sampled token: sampling_output → token_ids (for next pass)// and sampling_output → results[slot] (for callback)@@ -545,20 +537,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.token_copy_sampled.encode(sampling_output_buf_borrow.deref(),token_ids_buf_borrow.deref_mut(),- &mut command_buffer,+ &mut encoder,);let results_offset = slot * std::mem::size_of::<u32>();self.context.token_copy_results.encode(sampling_output_buf_borrow.deref(),(results_buffer.borrow_mut().deref_mut(), results_offset),- &mut command_buffer,+ &mut encoder,);// Scatter + register for all transformer layersself.context.cache_layers.borrow_mut().update_after_acceptance(&[0],None,- &mut command_buffer,+ &mut encoder,&self.context.kv_cache_update,);self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);@@ -575,7 +567,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {mask_buf_borrow.deref_mut(),update.unmask_col,update.mask_col,- &mut command_buffer,+ &mut encoder,);}}@@ -584,7 +576,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// Signal event for next passlet next_counter = current_counter + 1;- command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);+ encoder.encode_signal_event(&self.context.async_buffers.event, next_counter);self.context.async_buffers.counter.set(next_counter);// Add completion handler@@ -596,10 +588,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {on_complete(token);};- command_buffer.add_completion_handler(handler);+ encoder.add_completion_handler(handler);drop(token_ids_binding);- let pending = command_buffer.end_encoding().submit();+ let pending = encoder.end_encoding().submit();if should_capture {pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;@@ -825,27 +817,20 @@ impl<B: Backend> LanguageModelGenerator<B> {state: &mut ForwardPassState<B>,parameters: &EncodingParameters,sample: bool,- ) -> Result<<B::CommandBuffer as CommandBuffer>::Executable, Error> {- let mut command_buffer = self- .context- .context- .create_command_buffer()- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?- .start_encoding();+ ) -> Result<Executable<B>, Error> {+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;self.context.executables- .encode(state, parameters, &mut command_buffer)+ .encode(state, parameters, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;if sample {- self.context- .gpu_sampler- .encode(state, &mut command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ self.context.gpu_sampler.encode(state, &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;}- let executable = command_buffer.end_encoding();+ let executable = encoder.end_encoding();Ok(executable)}@@ -876,20 +861,20 @@ impl<B: Backend> LanguageModelGenerator<B> {suffix_start: Option<usize>,wait_until_completed: bool,) -> Result<(), Error> {- let mut command_buffer =- self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::<B>::new(self.context.context.as_ref())+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;{let mut cache_layers = self.context.cache_layers.borrow_mut();cache_layers.update_after_acceptance(accepted_token_indices,suffix_start,- &mut command_buffer,+ &mut encoder,&self.context.kv_cache_update,);}- let pending = command_buffer.end_encoding().submit();+ let pending = encoder.end_encoding().submit();if wait_until_completed {pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 848bcb579..62d18985d 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -18,10 +18,7 @@ use num_traits::NumCast;use crate::{ArrayElement, DataType,array::Array,- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- kernel::kv_cache_update::KVCacheUpdate,- },+ backends::common::{Backend, Encoder, kernel::kv_cache_update::KVCacheUpdate},classifier::Classifier,config::ModelMetadata,encodable_block::{EncodingParameters, Sampling},@@ -273,13 +270,13 @@ impl<B: Backend> TraceValidator<B> {None,);- let mut command_buffer =- ctx.context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder =+ Encoder::<B>::new(ctx.context.as_ref()).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;ctx.executables- .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)+ .encode(&mut state, &EncodingParameters::new(), &mut encoder).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- let pending = command_buffer.end_encoding().submit();+ let pending = encoder.end_encoding().submit();pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;let traces = state.traces().clone();diff --git a/crates/uzu/tests/integration/util/mod.rs b/crates/uzu/tests/integration/util/mod.rsindex 6b0e55f15..b67b341c8 100644--- a/crates/uzu/tests/integration/util/mod.rs+++ b/crates/uzu/tests/integration/util/mod.rs@@ -1,2 +1,3 @@+#[cfg_attr(not(feature = "tracing"), allow(dead_code))]pub mod path;pub mod speculator;diff --git a/crates/uzu/tests/performance/matmul/bench.rs b/crates/uzu/tests/performance/matmul/bench.rsindex 224a9aeb2..b6489d6d0 100644--- a/crates/uzu/tests/performance/matmul/bench.rs+++ b/crates/uzu/tests/performance/matmul/bench.rs@@ -4,8 +4,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending, Context,+ Backend, Encoder,kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},metal::Metal,@@ -39,7 +38,7 @@ fn encode_and_run(b_buffer: &Buf,d_buffer: &mut Buf,) -> Result<f64, BenchError> {- let mut command_buffer = context.create_command_buffer().map_err(|_| BenchError::CommandBuffer)?.start_encoding();+ let mut encoder = Encoder::new(context).map_err(|_| BenchError::CommandBuffer)?;kernel.encode(context,@@ -57,11 +56,10 @@ fn encode_and_run(leading_dimension_d: shape.output_dim as i32,transpose_b: true,},- &mut command_buffer,+ &mut encoder,);- let completed =- command_buffer.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;completed.gpu_execution_time().map(|duration| duration.as_secs_f64() * 1000.0).ok_or(BenchError::GpuTimestamps)}diff --git a/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rsindex 9afec7e7d..4fbaab822 100644--- a/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs@@ -8,7 +8,7 @@ use ndarray::{Array, Array3, s};use super::*;use crate::backends::{- common::{CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context},+ common::{Context, Encoder},metal::Metal,};@@ -104,8 +104,8 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {value_shape: [num_heads, seq_len, head_dim],};- let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();- match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut command_buffer) {+ let mut encoder = Encoder::new(context).unwrap();+ match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut encoder) {Ok(_) => {},Err(e) => {println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);@@ -113,7 +113,7 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {},}- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsindex e60458411..38e8c653f 100644--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs@@ -11,8 +11,7 @@ use crate::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context,+ Backend, Context, Encoder,gpu_types::QuantizationMode,kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,@@ -514,9 +513,9 @@ fn execute_quantized_matmul(output_dim,quantization_type,};- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();- kernel.encode(&mut command_buffer, args).unwrap();- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(ctx).unwrap();+ kernel.encode(&mut encoder, args).unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();}}@@ -534,9 +533,9 @@ fn execute_quantized_matmul(output_dim,quantization_type,};- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();- kernel.encode(&mut command_buffer, args).unwrap();- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(ctx).unwrap();+ kernel.encode(&mut encoder, args).unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();}let elapsed = start.elapsed();diff --git a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rsindex 83cf9fa95..a9955faed 100644--- a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs@@ -10,8 +10,7 @@ use crate::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels,+ Backend, Context, Encoder, Kernels,gpu_types::ActivationType,kernel::{Conv1dScanKernel,@@ -248,9 +247,9 @@ fn run_prefill_kernel_mode(head_dim: fixture.head_dim,};- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();- kernel.encode(&mut command_buffer, args, mode);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(ctx).unwrap();+ kernel.encode(&mut encoder, args, mode);+ encoder.end_encoding().submit().wait_until_completed().unwrap();let y_vec = read_buffer(&y_buf, fixture.total_x);let state_vec = read_buffer(&state_buf, fixture.total_state);@@ -318,7 +317,7 @@ fn run_conv_scan_once(write_buffer(&padded_buf, &host);}- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();+ let mut encoder = Encoder::new(ctx).unwrap();let mut b_out = y_buf.clone();let mut c_out = y_buf.clone();let mut state_out = scratch_buf.as_ref().unwrap_or(&state_buf).clone();@@ -338,17 +337,17 @@ fn run_conv_scan_once(channels as u32,0u32,ActivationType::SILU,- &mut command_buffer,+ &mut encoder,);if let Some(ref scratch) = scratch_buf {let bytes = channels * tap_count * size_of::<f32>();if bytes > 0 {- command_buffer.encode_copy(scratch, &mut state_buf, bytes);+ encoder.encode_copy(scratch, &mut state_buf, bytes);}}- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let y_vec = read_buffer(&y_buf, total_x);let state_vec = read_buffer(&state_buf, total_state);diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rsindex 86c4c5c66..9f070d94e 100644--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs@@ -12,8 +12,7 @@ use crate::{DataType,backends::{common::{- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending, Context, Kernels,+ Backend, Context, Encoder, Kernels,kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,attention::{AttentionGemmArguments, AttentionGemmBlock},@@ -317,7 +316,7 @@ fn run_single_pass_attention(.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -347,9 +346,9 @@ fn run_single_pass_attention(sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -386,7 +385,7 @@ fn run_single_pass_attention_with_is_causal(.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -415,9 +414,9 @@ fn run_single_pass_attention_with_is_causal(sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -453,7 +452,7 @@ fn run_gemm_attention(.new_buffer(num_heads * seq_len * head_dim * size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");let args = AttentionGemmArguments {queries_buffer: &query_buffer,@@ -473,8 +472,8 @@ fn run_gemm_attention(scale,};- kernel.encode(context, &mut command_buffer, args)?;- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ kernel.encode(context, &mut encoder, args)?;+ encoder.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -923,7 +922,7 @@ fn run_two_pass_attention().expect("Failed to create buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -954,7 +953,7 @@ fn run_two_pass_attention(mask_q_seq_stride,mask_head_stride,sinks_buffer.as_ref().map(|b| b),- &mut command_buffer,+ &mut encoder,);kernel_pass2.encode(&partials_buffer,@@ -963,9 +962,9 @@ fn run_two_pass_attention(&mut output_buffer,num_heads as u32,seq_len as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let output_ptr = output_buffer.contents().as_ptr() as *const f32;let output_slice = unsafe { std::slice::from_raw_parts(output_ptr, num_heads * seq_len * head_dim) };@@ -1158,7 +1157,7 @@ fn perf_two_pass_attention() {.expect("Failed to create buffer");// ---- Launch and time ----- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;@@ -1183,7 +1182,7 @@ fn perf_two_pass_attention() {None,None,sinks_buffer.as_ref().map(|b| b),- &mut command_buffer,+ &mut encoder,);kernel_pass2.encode(&partials_buffer,@@ -1192,11 +1191,11 @@ fn perf_two_pass_attention() {&mut output_buffer,num_heads as u32,suffix_length as u32,- &mut command_buffer,+ &mut encoder,);// Time both host-side and GPU executionlet host_timer = Instant::now();- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;match completed.gpu_execution_time() {diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rsindex 70ad296bf..329d2fbb3 100644--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs@@ -9,8 +9,7 @@ use crate::{DataType,backends::{common::{- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending, Context, Kernels,+ Backend, Context, Encoder, Kernels,kernel::{MinPKernel, TemperatureKernel,sampling::{ArgmaxStrategy, SamplingKernel},@@ -76,7 +75,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {.new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(&mut logits_buffer,@@ -88,10 +87,10 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {SamplingMethod::Greedy,batch_size,vocab_size,- &mut command_buffer,+ &mut encoder,).expect("Argmax sampling should succeed");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Check resultslet result_ptr = output_buffer.contents().as_ptr() as *const u32;@@ -166,7 +165,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {.expect("Failed to create buffer");// ---- Launch once and time ----- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(@@ -179,13 +178,13 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {SamplingMethod::Greedy,BATCH,VOCAB,- &mut command_buffer,+ &mut encoder,).expect("encode");// Time both host-side and GPU executionlet host_timer = Instant::now();- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;match completed.gpu_execution_time() {@@ -282,8 +281,7 @@ fn test_categorical_sampling() {.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(@@ -301,10 +299,10 @@ fn test_categorical_sampling() {},batch_size,vocab_size,- &mut command_buffer,+ &mut encoder,).expect("Categorical sampling should succeed");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let result_ptr = output_buffer.contents().as_ptr() as *const u32;let results = unsafe { std::slice::from_raw_parts(result_ptr, batch_size) };@@ -419,8 +417,7 @@ fn test_categorical_sampling_statistical() {.new_buffer_with_data(bytemuck::cast_slice(&seeds_vec), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(@@ -438,10 +435,10 @@ fn test_categorical_sampling_statistical() {},BATCH,VOCAB,- &mut command_buffer,+ &mut encoder,).expect("encode");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let result_ptr = output_buffer.contents().as_ptr() as *const u32;let results = unsafe { std::slice::from_raw_parts(result_ptr, BATCH) };@@ -521,7 +518,7 @@ fn perf_categorical_128k_vocab() {.expect("Failed to create buffer");// Launch and time- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(@@ -539,12 +536,12 @@ fn perf_categorical_128k_vocab() {},BATCH,VOCAB,- &mut command_buffer,+ &mut encoder,).expect("encode");let host_timer = Instant::now();- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;match completed.gpu_execution_time() {@@ -606,16 +603,9 @@ fn test_temperature_gpu_cpu_match() {.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(- Some(&logits_buffer),- &mut processed_buffer,- BATCH as u32,- VOCAB as u32,- TEMPERATURE,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();let gpu_ptr = processed_buffer.contents().as_ptr() as *const f32;let gpu_results = unsafe { std::slice::from_raw_parts(gpu_ptr, logits.len()) };@@ -673,9 +663,9 @@ fn test_minp_gpu_cpu_match() {.new_buffer(logits.len() * std::mem::size_of::<f32>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut command_buffer);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(Some(&logits_buffer), &mut processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();let results_ptr = processed_buffer.contents().as_ptr() as *const f32;let all_results = unsafe { std::slice::from_raw_parts(results_ptr, logits.len()) };@@ -756,8 +746,7 @@ fn test_minp_sampling_exact_match(.new_buffer_with_data(bytemuck::cast_slice(&seeds), metal::MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(&mut logits_buf,@@ -774,10 +763,10 @@ fn test_minp_sampling_exact_match(},batch_size,vocab_size,- &mut command_buffer,+ &mut encoder,).expect("encode");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let ptr = output_buf.contents().as_ptr() as *const u32;let sampled_ids = unsafe { std::slice::from_raw_parts(ptr, batch_size) };diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rsindex 01d5baa5f..074ca8fb6 100644--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs@@ -4,10 +4,7 @@ use crate::{DataType,array::ArrayContextExt,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, kernel::kv_cache_update::KVCacheUpdate,- },+ common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},metal::Metal,},forward_pass::kv_cache_layer::{INVALID_POSITION, KVCacheLayer, KVCacheLayerState},@@ -186,15 +183,15 @@ fn run_scenario(},};- let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();+ let mut encoder = Encoder::new(context).unwrap();layer.update_after_acceptance(&scenario.accepted_suffix_indices,scenario.suffix_start,- &mut command_buffer,+ &mut encoder,&kv_cache_update,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();layer.register_accepted_tokens(&scenario.accepted_token_positions);diff --git a/crates/uzu/tests/unit/kernel/activation_mul_test.rs b/crates/uzu/tests/unit/kernel/activation_mul_test.rsindex 5f4cba53b..f55d292e5 100644--- a/crates/uzu/tests/unit/kernel/activation_mul_test.rs+++ b/crates/uzu/tests/unit/kernel/activation_mul_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,- },+ common::{Backend, Context, Encoder, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel},cpu::Cpu,},};@@ -58,16 +55,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let fused_up_array = context.create_array_from(&[fused_len], &input.fused_up, "");let hidden_array = context.create_array_uninitialized(&[out_len], T::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(fused_up_array.buffer().borrow().deref(),hidden_array.buffer().borrow_mut().deref_mut(),input.h,input.m,input.act_type,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();hidden_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rsindex 7a9725809..e3216cd1b 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_gemm_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, kernel::attention::AttentionGemmArguments,- },+ common::{Backend, Context, Encoder, kernel::attention::AttentionGemmArguments},cpu::Cpu,},};@@ -148,10 +145,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {scale: input.scale,};- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- block.encode(&context, &mut command_buffer, args).expect("Failed to encode AttentionGemm");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ block.encode(&context, &mut encoder, args).expect("Failed to encode AttentionGemm");+ encoder.end_encoding().submit().wait_until_completed().unwrap();}output_array.as_slice().to_vec()diff --git a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rsindex b59aa67a0..8fc02f929 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_single_pass_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::AttentionSinglePassKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::AttentionSinglePassKernel},cpu::Cpu,},};@@ -133,7 +130,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let mask_q_seq_stride: Option<u32> = input.has_mask.then_some(input.sequence_length);let mask_head_stride: Option<u32> = input.has_mask.then_some(0);- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(queries_array.buffer().borrow().deref(),keys_array.buffer().borrow().deref(),@@ -153,9 +150,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {None::<&B::Buffer>,input.num_heads,input.suffix_length,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs b/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rsindex f06cbfda9..e16d1d23d 100644--- a/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/attention_update_kv_cache_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::AttentionUpdateKVCacheKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::AttentionUpdateKVCacheKernel},cpu::Cpu,},};@@ -53,7 +50,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let key_cache_array = context.create_array_from(&[cache_size], &input.key_cache, "");let value_cache_array = context.create_array_from(&[cache_size], &input.value_cache, "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(rotated_keys_buffer,qkv_array.buffer().borrow().deref(),@@ -65,9 +62,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,input.suffix_length,input.prefix_segment_length,input.max_sequence_length,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();(key_cache_array.as_slice().to_vec(), value_cache_array.as_slice().to_vec())}diff --git a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rsindex cde59daf1..25f3f9d6d 100644--- a/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs+++ b/crates/uzu/tests/unit/kernel/attention/sigmoid_gate_test.rs@@ -5,10 +5,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::SigmoidGateKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::SigmoidGateKernel},cpu::Cpu,},};@@ -32,15 +29,15 @@ fn get_output<T: ArrayElement + Float, B: Backend>(let gate_array = context.create_array_from(&[size], &gate_data.to_vec().into_boxed_slice(), "gate");let output_array = context.create_array_from(&[size], &output_data.to_vec().into_boxed_slice(), "output");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");let total_elements = config.suffix_length * config.num_heads * config.head_dim;kernel.encode(&*gate_array.buffer().borrow(),output_array.buffer().borrow_mut().deref_mut(),total_elements,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/audio/audio_test.rs b/crates/uzu/tests/unit/kernel/audio/audio_test.rsindex da86155f2..48aff9f65 100644--- a/crates/uzu/tests/unit/kernel/audio/audio_test.rs+++ b/crates/uzu/tests/unit/kernel/audio/audio_test.rs@@ -9,8 +9,7 @@ use uzu::{ArrayContextExt, DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels,+ Backend, Context, Encoder, Kernels,kernel::{AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,@@ -102,7 +101,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {}}- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input.buffer().borrow().deref(),weight.buffer().borrow().deref(),@@ -119,9 +118,9 @@ fn audio_conv1d_replicate_matches_reference_f32() {padding as i32,1_i32,batch_size as i32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let got = output.as_slice::<f32>();for index in 0..output_len {@@ -203,7 +202,7 @@ fn audio_causal_conv1d_matches_reference_f32() {}}- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input.buffer().borrow().deref(),weight.buffer().borrow().deref(),@@ -216,9 +215,9 @@ fn audio_causal_conv1d_matches_reference_f32() {kernel_size as i32,dilation as i32,batch_size as i32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let got = output.as_slice::<f32>();for index in 0..output_len {@@ -313,7 +312,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {}}- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input.buffer().borrow().deref(),weight.buffer().borrow().deref(),@@ -327,9 +326,9 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {stride as i32,groups as i32,batch_size as i32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let got = output.as_slice::<f32>();for index in 0..output_len {@@ -358,15 +357,15 @@ fn audio_leaky_relu_matches_reference_f32() {let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input.buffer().borrow().deref(),output.buffer().borrow_mut().deref_mut(),n as i32,slope,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let got = output.as_slice::<f32>();for index in 0..n {@@ -394,14 +393,9 @@ fn audio_tanh_matches_reference_f32() {let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();- kernel.encode(- input.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- n as i32,- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(input.buffer().borrow().deref(), output.buffer().borrow_mut().deref_mut(), n as i32, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let got = output.as_slice::<f32>();for index in 0..n {@@ -433,22 +427,22 @@ fn audio_add_and_scale_match_reference_f32() {let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");add_kernel.encode(a.buffer().borrow().deref(),b.buffer().borrow().deref(),sum.buffer().borrow_mut().deref_mut(),n as i32,- &mut command_buffer,+ &mut encoder,);scale_kernel.encode(sum.buffer().borrow().deref(),scaled.buffer().borrow_mut().deref_mut(),n as i32,scale_value,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let sum_got = sum.as_slice::<f32>();let scaled_got = scaled.as_slice::<f32>();@@ -489,16 +483,16 @@ fn audio_clamp_matches_reference_f32() {let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input.buffer().borrow().deref(),output.buffer().borrow_mut().deref_mut(),n as i32,min_value,max_value,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let got = output.as_slice::<f32>();for index in 0..n {@@ -534,7 +528,7 @@ fn audio_half_snake_matches_reference_f32() {let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input.buffer().borrow().deref(),alpha.buffer().borrow().deref(),@@ -545,9 +539,9 @@ fn audio_half_snake_matches_reference_f32() {negative_slope,eps,batch_size as i32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let got = output.as_slice::<f32>();for batch in 0..batch_size {@@ -600,7 +594,7 @@ fn audio_fsq_decode_matches_reference() {let output =context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(tokens.buffer().borrow().deref(),output.buffer().borrow_mut().deref_mut(),@@ -610,9 +604,9 @@ fn audio_fsq_decode_matches_reference() {codebook_dim as i32,&num_levels,batch_size as i32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let expected =reference_fsq_decode(&[0, 7, 11, 3, 5, 9], 2, batch_size, num_groups, seq_len, codebook_dim, &num_levels);@@ -658,7 +652,7 @@ fn audio_fsq_encode_matches_reference() {let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");- let mut command_buffer = context.create_command_buffer().expect("command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input.buffer().borrow().deref(),tokens.buffer().borrow_mut().deref_mut(),@@ -670,9 +664,9 @@ fn audio_fsq_encode_matches_reference() {&dim_base_index,eps,batch_size as i32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let expected = reference_fsq_encode(&input_values,diff --git a/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs b/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rsindex cbb24e518..d5e279fa1 100644--- a/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs+++ b/crates/uzu/tests/unit/kernel/full_precision_embedding_test.rs@@ -7,10 +7,7 @@ use half::{bf16, f16};use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement,- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- Kernels, kernel::FullPrecisionEmbeddingLookupKernel,- },+ backends::common::{Backend, Context, Encoder, Kernels, kernel::FullPrecisionEmbeddingLookupKernel},};struct Input<T: ArrayElement + Float> {@@ -62,7 +59,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let weights_array = context.create_array_from(&[input.vocab_size, input.model_dim], &input.weights, "");let output_array = context.create_array_uninitialized(&[input.batch_size, input.model_dim], T::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to get command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to get encoder");kernel.encode(token_ids_array.buffer().borrow().deref(),weights_array.buffer().borrow().deref(),@@ -71,9 +68,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {input.vocab_size as u32,input.model_dim as u32,input.input_scale,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rsindex 444f0b17c..4fdfc6002 100644--- a/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/gemv_test.rs@@ -9,8 +9,7 @@ use uzu::{ArrayContextExt, ArrayElement,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context,+ Backend, Context, Encoder,kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},cpu::Cpu,@@ -68,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type()).expect("Failed to create MatmulKernel");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(&context,MatmulArguments {@@ -85,9 +84,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {leading_dimension_d: n,transpose_b: true,},- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();drop(d_ref);d_array.as_slice().to_vec()diff --git a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rsindex c868918d6..a0772b176 100644--- a/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/matmul_correctness_test.rs@@ -10,8 +10,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context,+ Backend, Context, Encoder,kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},metal::Metal,@@ -49,7 +48,7 @@ fn run_metal_matmul(let mut kernel =<<Metal as Backend>::Kernels as MatmulKernels>::MatmulKernel::new(ctx, DataType::BF16).expect("kernel");- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();+ let mut encoder = Encoder::new(ctx).unwrap();kernel.encode(ctx,MatmulArguments {@@ -66,9 +65,9 @@ fn run_metal_matmul(leading_dimension_d: n as i32,transpose_b,},- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();unsafe {let ptr = d_buf.contents().as_ptr() as *const bf16;diff --git a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rsindex 103c5d53a..e2a476fdf 100644--- a/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/steel_gemm_test.rs@@ -9,8 +9,7 @@ use uzu::{ArrayContextExt, ArrayElement,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context,+ Backend, Context, Encoder,kernel::matmul::{MatmulArguments, MatmulKernel, MatmulKernels},},cpu::Cpu,@@ -68,7 +67,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let mut kernel = <B::Kernels as MatmulKernels>::MatmulKernel::new(&context, T::data_type()).expect("Failed to create MatmulKernel");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(&context,MatmulArguments {@@ -85,9 +84,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {leading_dimension_d: n,transpose_b: true,},- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();drop(d_ref);d_array.as_slice().to_vec()diff --git a/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rsindex ce4c5badd..acc5932cb 100644--- a/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/counts_offsets_fused_test.rs@@ -8,8 +8,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel},},metal::Metal,@@ -76,7 +75,7 @@ fn gen_topk_ids_from_logits(let mut topk_probs_buf = alloc_buffer::<bf16>(ctx, t * k);// Use fused router+topk kernel- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16).expect("router_topk kernel");router_topk.encode(@@ -90,9 +89,9 @@ fn gen_topk_ids_from_logits(e as u32,k as u32,true,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let ids_ptr = topk_ids_buf.contents().as_ptr() as *const i32;let ids = unsafe { std::slice::from_raw_parts(ids_ptr, t * k) };@@ -124,8 +123,7 @@ fn test_counts_offsets_fused_parity_random() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let mut command_buffer =- ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");kernel.encode(&topk_ids_buf,@@ -135,10 +133,10 @@ fn test_counts_offsets_fused_parity_random() {t as u32,e as u32,k as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Verify offsetslet offsets_ptr = offsets_buf.contents().as_ptr() as *const u32;@@ -174,7 +172,7 @@ fn test_counts_offsets_fused_edge_cases() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -183,9 +181,9 @@ fn test_counts_offsets_fused_edge_cases() {t as u32,e as u32,k as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };@@ -204,7 +202,7 @@ fn test_counts_offsets_fused_edge_cases() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -213,9 +211,9 @@ fn test_counts_offsets_fused_edge_cases() {t as u32,e as u32,k as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let offsets_gpu = unsafe { std::slice::from_raw_parts(offsets_buf.contents().as_ptr() as *const u32, e + 1) };assert!(offsets_gpu.iter().all(|&v| v == 0));diff --git a/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rsindex 8b038f7f0..e3bceb55d 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_block_e2e_test.rs@@ -7,8 +7,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,@@ -349,7 +348,7 @@ fn run_moe_parity_test_internal(// Encode ALL kernels in one command buffereprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");// Router + TopK (fused kernel)let router_topk =@@ -365,7 +364,7 @@ fn run_moe_parity_test_internal(e as u32,k as u32,true,- &mut command_buffer,+ &mut encoder,);let fused_kernel =@@ -378,7 +377,7 @@ fn run_moe_parity_test_internal(t as u32,e as u32,k as u32,- &mut command_buffer,+ &mut encoder,);let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)@@ -391,7 +390,7 @@ fn run_moe_parity_test_internal(num_blocks as u32,num_tiles as u32,0u32,- &mut command_buffer,+ &mut encoder,);let scatter_map_kernel =@@ -411,12 +410,12 @@ fn run_moe_parity_test_internal(num_blocks as u32,num_tiles as u32,&mut tok2row_buf,- &mut command_buffer,+ &mut encoder,);let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");gather.encode(- &mut command_buffer,+ &mut encoder,DataType::BF16,MoeGatherArguments {x_buffer: &x_buf,@@ -464,7 +463,7 @@ fn run_moe_parity_test_internal(silu_alpha,data_type: DataType::BF16,};- experts.encode(&mut command_buffer, args);+ experts.encode(&mut encoder, args);let finalize =<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");@@ -476,11 +475,11 @@ fn run_moe_parity_test_internal(t as u32,d_model as u32,k as u32,- &mut command_buffer,+ &mut encoder,);eprintln!("[E2E] All kernels encoded. Committing ONCE and waiting...");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();eprintln!("[E2E] GPU execution completed");// Read GPU outputdiff --git a/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rsindex b7c5377da..844a5d72b 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_experts_perf_test.rs@@ -8,8 +8,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context,+ Backend, Encoder,kernel::moe::{MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,MoeExpertsTwoPassDecodeBlock, MoeExpertsTwoPassPrefillBlock,@@ -108,9 +107,9 @@ fn run_decode_case(let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);for _ in 0..warmup {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");experts_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -140,15 +139,15 @@ fn run_decode_case(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");experts_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -178,7 +177,7 @@ fn run_decode_case(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -270,7 +269,7 @@ fn run_two_pass_prefill_case(let mut row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);for _ in 0..warmup {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -299,14 +298,14 @@ fn run_two_pass_prefill_case(silu_alpha: 1.702,data_type: DataType::BF16,};- experts_kernel.encode(&mut command_buffer, args);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ experts_kernel.encode(&mut encoder, args);+ encoder.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -335,8 +334,8 @@ fn run_two_pass_prefill_case(silu_alpha: 1.702,data_type: DataType::BF16,};- experts_kernel.encode(&mut command_buffer, args);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ experts_kernel.encode(&mut encoder, args);+ encoder.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -411,9 +410,9 @@ fn run_fused_single_token_case(let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);for _ in 0..warmup {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");fused_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -436,15 +435,15 @@ fn run_fused_single_token_case(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");fused_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -467,7 +466,7 @@ fn run_fused_single_token_case(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -589,9 +588,9 @@ fn run_indirect_decode_timed(let mut row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);for _ in 0..warmup {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");experts_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -621,15 +620,15 @@ fn run_indirect_decode_timed(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");experts_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -659,7 +658,7 @@ fn run_indirect_decode_timed(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}@@ -706,9 +705,9 @@ fn run_fused_decode_timed(let mut y_buf = alloc_buffer::<bf16>(ctx, d_model);for _ in 0..warmup {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");fused_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -731,15 +730,15 @@ fn run_fused_decode_timed(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();}let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");fused_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -762,7 +761,7 @@ fn run_fused_decode_timed(data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();times.push(start.elapsed().as_secs_f64() * 1000.0);}diff --git a/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rsindex 701784cb9..2a3117213 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_experts_test.rs@@ -15,7 +15,7 @@ use uzu::{DataType,backends::{common::{- CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Encoder,kernel::moe::{MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeBlock,MoeExpertsTwoPassPrefillBlock,@@ -419,13 +419,13 @@ fn test_two_pass_decode_correctness() {// Execute 2-pass decode kernellet experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");const K_TILE: usize = 64;let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;experts_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -456,7 +456,7 @@ fn test_two_pass_decode_correctness() {},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Read GPU partial output and do CPU finalize (weighted sum)let y_partial_gpu =@@ -562,9 +562,9 @@ fn test_two_pass_decode_multi_token() {let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");experts_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -594,7 +594,7 @@ fn test_two_pass_decode_multi_token() {data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let y_partial_gpu =unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };@@ -663,7 +663,7 @@ fn test_two_pass_prefill_correctness() {let mut dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");let args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -692,8 +692,8 @@ fn test_two_pass_prefill_correctness() {silu_alpha,data_type: DataType::BF16,};- experts_kernel.encode(&mut command_buffer, args);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ experts_kernel.encode(&mut encoder, args);+ encoder.end_encoding().submit().wait_until_completed().unwrap();let y_partial_gpu =unsafe { std::slice::from_raw_parts(y_partial_buf.contents().as_ptr() as *const bf16, sum_k * d_model) };@@ -791,10 +791,10 @@ fn test_fused_single_token_decode() {// Run fused decode kernellet fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");fused_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -818,7 +818,7 @@ fn test_fused_single_token_decode() {},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Read GPU outputlet y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };@@ -914,9 +914,9 @@ fn test_fused_single_token_k4() {let mut y_buf = alloc_buffer::<bf16>(&ctx, d_model);let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");fused_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -939,7 +939,7 @@ fn test_fused_single_token_k4() {data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let y_gpu = unsafe { std::slice::from_raw_parts(y_buf.contents().as_ptr() as *const bf16, d_model) };diff --git a/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rsindex 583b5e376..5f9c1c975 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_finalize_test.rs@@ -6,10 +6,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::MoeFinalizeKernel,- },+ common::{Backend, Encoder, Kernels, kernel::MoeFinalizeKernel},metal::Metal,},};@@ -93,7 +90,7 @@ fn test_finalize_correctness() {// Execute finalize kernellet finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize kernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");finalize.encode(&tok2row_buf,&probs_buf,@@ -102,9 +99,9 @@ fn test_finalize_correctness() {t as u32,d_model as u32,k as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Comparelet y_gpu = unsafe { std::slice::from_raw_parts(y_out_buf.contents().as_ptr() as *const bf16, t * d_model) };diff --git a/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rsindex b38c936ef..5e4bf1399 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_gather_test.rs@@ -7,7 +7,7 @@ use uzu::{DataType,backends::{common::{- CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Encoder,kernel::moe::{MoeGatherArguments, MoeGatherKernels},},metal::Metal,@@ -81,9 +81,9 @@ fn test_gather_correctness() {// Execute gather kernel using kernel structlet gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");gather.encode(- &mut command_buffer,+ &mut encoder,DataType::BF16,MoeGatherArguments {x_buffer: &x_buf,@@ -95,7 +95,7 @@ fn test_gather_correctness() {d_model,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Comparelet x_gpu =diff --git a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rsindex e87707aba..01a5f181f 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs@@ -8,8 +8,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,@@ -123,8 +122,7 @@ fn test_moe_e2e_decode_perf() {// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {- let mut command_buffer =- ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");router_topk.encode(&x_buf,&router_w_buf,@@ -136,9 +134,9 @@ fn test_moe_e2e_decode_perf() {e as u32,k as u32,true,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});fused_perf.print();@@ -182,8 +180,7 @@ fn test_moe_e2e_prefill_perf() {// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {- let mut command_buffer =- ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");router_topk.encode(&x_buf,&router_w_buf,@@ -195,9 +192,9 @@ fn test_moe_e2e_prefill_perf() {e as u32,k as u32,true,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});fused_perf.print();@@ -309,7 +306,7 @@ fn test_moe_pipeline_breakdown_decode() {// Testing: Router + TopK + Counts+Offsets (FUSED)let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");router_topk_fused_kernel.encode(&x_buf,&router_w_buf,@@ -321,13 +318,13 @@ fn test_moe_pipeline_breakdown_decode() {e as u32,k as u32,true,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");counts_offsets_kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -336,13 +333,13 @@ fn test_moe_pipeline_breakdown_decode() {t as u32,e as u32,k as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});let scatter_perf = time_kernel("Scatter", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");scatter_bases_kernel.encode(&partials_buf,&mut block_bases_buf,@@ -351,7 +348,7 @@ fn test_moe_pipeline_breakdown_decode() {num_blocks as u32,num_tiles as u32,0u32,- &mut command_buffer,+ &mut encoder,);scatter_map_kernel.encode(&topk_ids_buf,@@ -367,15 +364,15 @@ fn test_moe_pipeline_breakdown_decode() {num_blocks as u32,num_tiles as u32,&mut tok2row_buf,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});let gather_perf = time_kernel("Gather", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");gather_kernel.encode(- &mut command_buffer,+ &mut encoder,DataType::BF16,MoeGatherArguments {x_buffer: &x_buf,@@ -387,13 +384,13 @@ fn test_moe_pipeline_breakdown_decode() {d_model,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");experts_kernel.encode(- &mut command_buffer,+ &mut encoder,MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,@@ -423,11 +420,11 @@ fn test_moe_pipeline_breakdown_decode() {data_type: DataType::BF16,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});let finalize_perf = time_kernel("Finalize", 2, 5, || {- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");finalize_kernel.encode(&tok2row_buf,&topk_probs_buf,@@ -436,9 +433,9 @@ fn test_moe_pipeline_breakdown_decode() {t as u32,d_model as u32,k as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();});// Print resultsdiff --git a/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rsindex 75100831f..2f73636a0 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_router_topk_test.rs@@ -6,10 +6,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::MoeRouterTopKKernel,- },+ common::{Backend, Encoder, Kernels, kernel::MoeRouterTopKKernel},metal::Metal,},};@@ -160,7 +157,7 @@ fn run_router_topk_once(// For BFloat16 kernel, probs buffer must be bf16, not f32let mut probs_buf = alloc_buffer::<bf16>(ctx, t * k);- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx).expect("Failed to create encoder");kernel.encode(&input_buf,&weight_buf,@@ -172,9 +169,9 @@ fn run_router_topk_once(e as u32,k as u32,renorm,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let ids_ptr = ids_buf.contents().as_ptr() as *const i32;let probs_ptr = probs_buf.contents().as_ptr() as *const bf16;diff --git a/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rsindex e532d764a..febdb4e2b 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_scatter_test.rs@@ -7,8 +7,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels,+ Backend, Encoder, Kernels,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeRouterTopKKernel,MoeScatterBucketsKernel,@@ -85,7 +84,7 @@ fn test_scatter_buckets_parity() {// Use fused router+topk kernellet router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router_topk");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");router_topk.encode(&input_buf,&weight_buf,@@ -97,9 +96,9 @@ fn test_scatter_buckets_parity() {e as u32,k as u32,true,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Read back CPU probs for reference comparelet probs_bf16 =@@ -114,7 +113,7 @@ fn test_scatter_buckets_parity() {let fused_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");fused_kernel.encode(&topk_ids_buf,&mut offsets_buf,@@ -123,9 +122,9 @@ fn test_scatter_buckets_parity() {t as u32,e as u32,k as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Partials already created by fused kernel abovelet num_blocks = 1; // Fused kernel uses single block@@ -138,7 +137,7 @@ fn test_scatter_buckets_parity() {let scatter_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel::new(&ctx, DataType::BF16).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsKernel");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");let mut block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);scatter_bases_kernel.encode(@@ -149,16 +148,16 @@ fn test_scatter_buckets_parity() {num_blocks as u32,num_tiles as u32,0u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let sumk = unsafe { std::slice::from_raw_parts(sumk_buf.contents().as_ptr() as *const u32, 1) }[0] as usize;let mut out_ids_buf = alloc_buffer::<i32>(&ctx, sumk);let mut out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");scatter_kernel.encode(&topk_ids_buf,&topk_probs_buf,@@ -172,10 +171,10 @@ fn test_scatter_buckets_parity() {k as u32,num_blocks as u32,num_tiles as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let out_ids = unsafe { std::slice::from_raw_parts(out_ids_buf.contents().as_ptr() as *const i32, sumk) };let out_probs_h = unsafe { std::slice::from_raw_parts(out_probs_buf.contents().as_ptr() as *const bf16, sumk) };diff --git a/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rsindex ab7eac9f1..57c6e6e19 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_tiles_test.rs@@ -4,7 +4,7 @@ use metal::MTLBuffer;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::backends::{common::{- CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ Encoder,kernel::moe::{MoeTileCountsArguments, MoeTileMapKernels, MoeTileScanArguments},},metal::Metal,@@ -38,16 +38,16 @@ fn test_tile_counts_correctness() {// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");tile_kernel.encode_counts(- &mut command_buffer,+ &mut encoder,MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &mut tile_counts_buf,e,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Comparelet tile_counts_gpu =@@ -82,9 +82,9 @@ fn test_tile_scan_correctness() {// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");tile_kernel.encode_scan(- &mut command_buffer,+ &mut encoder,MoeTileScanArguments {tile_counts_buffer: &tile_counts_buf,tile_offsets_buffer: &mut tile_offsets_buf,@@ -92,7 +92,7 @@ fn test_tile_scan_correctness() {e,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();// Comparelet tile_offsets_gpu =@@ -121,16 +121,16 @@ fn test_tile_edge_cases() {let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");tile_kernel.encode_counts(- &mut command_buffer,+ &mut encoder,MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &mut tile_counts_buf,e,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let tile_counts_gpu =unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };@@ -151,16 +151,16 @@ fn test_tile_edge_cases() {let mut tile_counts_buf = alloc_buffer::<u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let mut command_buffer = ctx.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");tile_kernel.encode_counts(- &mut command_buffer,+ &mut encoder,MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &mut tile_counts_buf,e,},);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let tile_counts_gpu =unsafe { std::slice::from_raw_parts(tile_counts_buf.contents().as_ptr() as *const u32, e) };diff --git a/crates/uzu/tests/unit/kernel/qk_norm_test.rs b/crates/uzu/tests/unit/kernel/qk_norm_test.rsindex ad6d5b038..ebf13d876 100644--- a/crates/uzu/tests/unit/kernel/qk_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/qk_norm_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::QKNormKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::QKNormKernel},cpu::Cpu,},};@@ -132,7 +129,7 @@ fn get_output<let qkv_array = context.create_array_from(&[qkv_len], &input.qkv, "");let scales_array = context.create_array_from(&[input.scales.len()], &input.scales, "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(None::<&B::Buffer>,scales_array.buffer().borrow().deref(),@@ -146,10 +143,9 @@ fn get_output<input.head_offset,input.head_count,input.full_layer,- &mut command_buffer,+ &mut encoder,);-- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");qkv_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rsindex db9891942..75d5c0158 100644--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs@@ -10,10 +10,7 @@ use rand::{RngExt, SeedableRng, prelude::StdRng};use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending, Context, Kernels, kernel::RMSNormKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::RMSNormKernel},cpu::Cpu,},};@@ -144,7 +141,7 @@ fn get_output<false => context.create_array_uninitialized(&[input_size], OutputT::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input_buffer,scales_array.buffer().borrow().deref(),@@ -154,12 +151,11 @@ fn get_output<input.epsilon,input.scale_offset,input.full_layer,- &mut command_buffer,+ &mut encoder,);let instant = Instant::now();- let completed =- command_buffer.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ let completed = encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");let host_elapsed_ms = instant.elapsed().as_secs_f64() * 1e3;let gpu_elapsed_ms = completed.gpu_execution_time();diff --git a/crates/uzu/tests/unit/kernel/rope_test.rs b/crates/uzu/tests/unit/kernel/rope_test.rsindex 114d55fd7..24a306243 100644--- a/crates/uzu/tests/unit/kernel/rope_test.rs+++ b/crates/uzu/tests/unit/kernel/rope_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::RopeKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::RopeKernel},cpu::Cpu,},};@@ -93,7 +90,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let rotated_queries_array = context.create_array_uninitialized(&[queries_len], T::data_type(), "");let rotated_keys_array = context.create_array_uninitialized(&[keys_len], T::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(qkv_array.buffer().borrow().deref(),cosines_array.buffer().borrow().deref(),@@ -107,9 +104,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,input.num_groups,input.suffix_length,input.max_sequence_length,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();(rotated_queries_array.as_slice().to_vec(), rotated_keys_array.as_slice().to_vec())}diff --git a/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs b/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rsindex b64f31870..35c3f92cf 100644--- a/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs+++ b/crates/uzu/tests/unit/kernel/sampling/argmax_test_old.rs@@ -9,8 +9,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- Kernels,+ Backend, Context, Encoder, Kernels,gpu_types::ArgmaxPair,kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},},@@ -71,16 +70,16 @@ fn get_output_single<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vlet logits_array = context.create_array_from(&[len], &input.logits, "");let output_array = context.create_array_uninitialized(&[input.batch_size as usize], DataType::U32, "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(logits_array.buffer().borrow().deref(),output_array.buffer().borrow_mut().deref_mut(),input.batch_size,input.vocab_size,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice::<u32>().to_vec()}@@ -106,23 +105,23 @@ fn get_output_two_pass<T: ArrayElement + Float, B: Backend>(input: &Input<T>) ->.create_buffer(partial_results_count * size_of::<ArgmaxPair>()).expect("Failed to create partial results buffer");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");main_kernel.encode(logits_array.buffer().borrow().deref(),&mut partial_results_buffer,input.batch_size,input.vocab_size,- &mut command_buffer,+ &mut encoder,);final_kernel.encode(&partial_results_buffer,output_array.buffer().borrow_mut().deref_mut(),input.batch_size,input.vocab_size,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice::<u32>().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs b/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rsindex 4fbdce850..5e293cd2a 100644--- a/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/gumbel_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::GumbelKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::GumbelKernel},cpu::Cpu,},gumbel_float, revidx,@@ -71,17 +68,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {false => context.create_array_uninitialized(&[len], T::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(logits_buffer,seeds_array.buffer().borrow().deref(),output_array.buffer().borrow_mut().deref_mut(),input.batch_size,input.vocab_size,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs b/crates/uzu/tests/unit/kernel/sampling/temperature_test.rsindex 77f2c4944..d8d502c23 100644--- a/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/temperature_test.rs@@ -8,10 +8,7 @@ use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::TemperatureKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::TemperatureKernel},cpu::Cpu,},};@@ -68,17 +65,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {false => context.create_array_uninitialized(&[len], T::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(logits_buffer,output_array.buffer().borrow_mut().deref_mut(),input.batch_size,input.vocab_size,input.temperature,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rsindex dfbbcb86d..993664f96 100644--- a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs+++ b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs@@ -7,8 +7,7 @@ use rand_distr::Normal;use uzu::{ArrayElement,backends::common::{- Backend, Buffer, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels,+ Backend, Buffer, Context, Encoder, Kernels,gpu_types::ArgmaxPair,kernel::{ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel},},@@ -55,29 +54,23 @@ fn do_argmax_backend<B: Backend, T: ArrayElement + Float>(let twopass_kernel_main = <B::Kernels as Kernels>::ArgmaxMainKernel::new(context, T::data_type()).unwrap();let twopass_kernel_final = <B::Kernels as Kernels>::ArgmaxFinalKernel::new(context).unwrap();- let mut command_buffer = context.create_command_buffer().unwrap().start_encoding();- single_kernel.encode(- &logits_buffer,- &mut single_output_buffer,- batch_size as u32,- vocab_size as u32,- &mut command_buffer,- );+ let mut encoder = Encoder::new(context).unwrap();+ single_kernel.encode(&logits_buffer, &mut single_output_buffer, batch_size as u32, vocab_size as u32, &mut encoder);twopass_kernel_main.encode(&logits_buffer,&mut twopass_partial_results_buffer,batch_size as u32,vocab_size as u32,- &mut command_buffer,+ &mut encoder,);twopass_kernel_final.encode(&twopass_partial_results_buffer,&mut twopass_output_buffer,batch_size as u32,vocab_size as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let single_output = unsafe {std::slice::from_raw_parts(single_output_buffer.cpu_ptr().as_ptr() as *const u32, batch_size)diff --git a/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_k_test.rsindex 39aaad087..b8ef4d709 100644--- a/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/top_k_test.rs@@ -9,10 +9,7 @@ use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{- common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::TopKKernel,- },+ common::{Backend, Context, Encoder, Kernels, kernel::TopKKernel},cpu::Cpu,},};@@ -68,17 +65,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {false => context.create_array_uninitialized(&[len], T::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(logits_buffer,output_array.buffer().borrow_mut().deref_mut(),input.batch_size,input.vocab_size,input.top_k,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rsindex 8cabf92ad..0e6a47fd5 100644--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs@@ -11,8 +11,7 @@ use uzu::{ArrayContextExt, ArrayElement, DataType,backends::{common::{- Backend, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial,- CommandBufferPending, Context, Kernels,+ Backend, Context, Encoder, Kernels,kernel::{TopPKernel, sampling::SamplingKernel},},cpu::Cpu,@@ -73,17 +72,17 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {false => context.create_array_uninitialized(&[len], T::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(logits_buffer,output_array.buffer().borrow_mut().deref_mut(),input.batch_size,input.vocab_size,input.top_p,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}@@ -280,8 +279,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; batch_size];let seeds_array = context.create_array_from(&[batch_size], &seeds, "");- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(logits_array.buffer().borrow_mut().deref_mut(),@@ -298,10 +296,10 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(},batch_size,vocab_size,- &mut command_buffer,+ &mut encoder,).expect("encode");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let sampled_ids: &[u32] = output_array.as_slice();@@ -384,8 +382,7 @@ fn test_topp_sampling_statistical_large() {let seeds: Vec<u64> = vec![TEST_SAMPLING_SEED + draw as u64; BATCH];let seeds_array = context.create_array_from(&[BATCH], &seeds, "");- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(@@ -403,10 +400,10 @@ fn test_topp_sampling_statistical_large() {},BATCH,VOCAB,- &mut command_buffer,+ &mut encoder,).expect("encode");- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let sample_ids: &[u32] = output_array.as_slice();for (b, &tok) in sample_ids.iter().enumerate() {@@ -461,8 +458,7 @@ fn perf_topp_128k_vocab() {let seeds_array = context.create_array_from(&[BATCH], &seeds, "");let output_array = context.create_array_uninitialized(&[BATCH], DataType::U32, "");- let mut command_buffer =- context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(@@ -480,12 +476,12 @@ fn perf_topp_128k_vocab() {},BATCH,VOCAB,- &mut command_buffer,+ &mut encoder,).expect("encode");let host_timer = std::time::Instant::now();- let completed = command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;match completed.gpu_execution_time() {diff --git a/crates/uzu/tests/unit/kernel/ssd_update_test.rs b/crates/uzu/tests/unit/kernel/ssd_update_test.rsindex b057715b8..84c885c75 100644--- a/crates/uzu/tests/unit/kernel/ssd_update_test.rs+++ b/crates/uzu/tests/unit/kernel/ssd_update_test.rs@@ -7,8 +7,7 @@ use uzu::{DataType,backends::{common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::SSDUpdateKernel,+ Backend, Context, Encoder, Kernels, kernel::SSDUpdateKernel,},metal::Metal,},@@ -242,7 +241,7 @@ fn ssd_update_with_z_bf16() {.expect("Failed to create buffer");let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16, false).unwrap();- let mut command_buffer = ctx.create_command_buffer().unwrap().start_encoding();+ let mut encoder = Encoder::new(ctx.as_ref()).unwrap();kernel.encode(&x_buf,&dt_buf,@@ -262,9 +261,9 @@ fn ssd_update_with_z_bf16() {bsz as u32,h as u32,dh as u32,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();let y_ptr = y_buf.contents().as_ptr() as *const bf16;let y_out = unsafe { std::slice::from_raw_parts(y_ptr, bsz * h * dh) };diff --git a/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rsindex c77ce3f45..0b90c6239 100644--- a/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs+++ b/crates/uzu/tests/unit/kernel/tensor_add_bias_test.rs@@ -7,10 +7,7 @@ use half::{bf16, f16};use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement,- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- Kernels, kernel::TensorAddBiasKernel,- },+ backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},};struct Input<T: ArrayElement + Float> {@@ -77,16 +74,16 @@ fn get_output<T: ArrayElement + Float, B: Backend>(false => context.create_array_uninitialized(&[size], T::data_type(), ""),};- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(input_buffer,bias_array.buffer().borrow_mut().deref(),output_array.buffer().borrow_mut().deref_mut(),input.num_cols,input.length,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();output_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs b/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rsindex 10a880822..f1179f9e3 100644--- a/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs+++ b/crates/uzu/tests/unit/kernel/tensor_add_swap_test.rs@@ -4,10 +4,7 @@ use half::{bf16, f16};use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement,- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- Kernels, kernel::TensorAddSwapKernel,- },+ backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddSwapKernel},};struct Input<T: ArrayElement + Float> {@@ -48,14 +45,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> (Vec<T>,let skip_array = context.create_array_from(&[size], &input.skip_buffer, "");let main_array = context.create_array_from(&[size], &input.main_buffer, "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(skip_array.buffer().borrow_mut().deref_mut(),main_array.buffer().borrow_mut().deref_mut(),input.length,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();(skip_array.as_slice().to_vec(), main_array.as_slice().to_vec())}diff --git a/crates/uzu/tests/unit/kernel/tensor_copy_test.rs b/crates/uzu/tests/unit/kernel/tensor_copy_test.rsindex 2af7a1929..5fb070f58 100644--- a/crates/uzu/tests/unit/kernel/tensor_copy_test.rs+++ b/crates/uzu/tests/unit/kernel/tensor_copy_test.rs@@ -7,10 +7,7 @@ use half::{bf16, f16};use num_traits::Float;use uzu::{ArrayContextExt, ArrayElement,- backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- Kernels, kernel::TensorCopyKernel,- },+ backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorCopyKernel},};struct Input<T: ArrayElement + Float> {@@ -47,14 +44,14 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let src_array = context.create_array_from(&[size], &input.src, "");let dst_array = context.create_array_uninitialized(&[size], T::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");kernel.encode(src_array.buffer().borrow().deref(),dst_array.buffer().borrow_mut().deref_mut(),input.length,- &mut command_buffer,+ &mut encoder,);- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ encoder.end_encoding().submit().wait_until_completed().unwrap();dst_array.as_slice().to_vec()}diff --git a/crates/uzu/tests/unit/kernel/token_copy_test.rs b/crates/uzu/tests/unit/kernel/token_copy_test.rsindex d64315950..7ef483573 100644--- a/crates/uzu/tests/unit/kernel/token_copy_test.rs+++ b/crates/uzu/tests/unit/kernel/token_copy_test.rs@@ -3,8 +3,7 @@ use std::ops::{Deref, DerefMut};use uzu::{ArrayContextExt, ArrayElement,backends::common::{- Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,- Kernels,+ Backend, Context, Encoder, Kernels,kernel::{TokenCopySampledKernel, TokenCopyToResultsKernel},},};@@ -18,13 +17,9 @@ fn test_token_copy_sampled_impl<B: Backend>(src_value: u32) {let src_array = context.create_array_from(&[1], &[src_value], "");let dst_array = context.create_array_uninitialized(&[1], u64::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(- src_array.buffer().borrow().deref(),- dst_array.buffer().borrow_mut().deref_mut(),- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();let output = dst_array.as_slice::<u64>().to_vec();assert_eq!(output[0], src_value as u64, "TokenCopySampled failed for backend {}", std::any::type_name::<B>());@@ -39,13 +34,9 @@ fn test_token_copy_to_results_impl<B: Backend>(src_value: u32) {let src_array = context.create_array_from(&[1], &[src_value], "");let dst_array = context.create_array_uninitialized(&[1], u32::data_type(), "");- let mut command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();- kernel.encode(- src_array.buffer().borrow().deref(),- dst_array.buffer().borrow_mut().deref_mut(),- &mut command_buffer,- );- command_buffer.end_encoding().submit().wait_until_completed().unwrap();+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(src_array.buffer().borrow().deref(), dst_array.buffer().borrow_mut().deref_mut(), &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();let output = dst_array.as_slice::<u32>().to_vec();assert_eq!(output[0], src_value, "TokenCopyToResults failed for backend {}", std::any::type_name::<B>());diff --git a/crates/uzu/tests_unit/hazard_tracker_test.rs b/crates/uzu/tests_unit/hazard_tracker_test.rsnew file mode 100644index 000000000..b3d51d89b--- /dev/null+++ b/crates/uzu/tests_unit/hazard_tracker_test.rs@@ -0,0 +1,46 @@+use super::*;++#[test]+fn test_hazard_tracker_simple_compute_sequence() {+ let mut ht = HazardTracker::new();++ // RAR is fine (many readers don't interfere with each other)+ for _ in 0..123 {+ assert_eq!(+ ht.access(&[Access {+ range: 0..16,+ flags: AccessFlags::compute_read()+ }]),+ None+ );+ }++ // WAR causes barrier after reads before writes (to ensure that the reads are done before writes can rewrite the data)+ assert_eq!(+ ht.access(&[Access {+ range: 0..16,+ flags: AccessFlags::compute_write()+ }]),+ Some((AccessFlags::compute_read(), AccessFlags::compute_write()))+ );++ // WAW causes barrier after writes before writes (to ensure the last write wins)+ for _ in 0..123 {+ assert_eq!(+ ht.access(&[Access {+ range: 0..16,+ flags: AccessFlags::compute_write()+ }]),+ Some((AccessFlags::compute_write(), AccessFlags::compute_write()))+ );+ }++ // RAW causes barrier after writes before reads (to ensure that writes wrote the data before reads read it)+ assert_eq!(+ ht.access(&[Access {+ range: 0..16,+ flags: AccessFlags::compute_read()+ }]),+ Some((AccessFlags::compute_write(), AccessFlags::compute_read()))+ );+}