Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

more muts (#229)

uuuvn98fd54b197bb
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rs
index 537c7f9c4..b7d52f3d5 100644
--- a/crates/uzu/build/common/traitgen.rs
+++ b/crates/uzu/build/common/traitgen.rs
@@ -69,8 +69,8 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {
fn new(context: &<Self::Backend as Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as Backend>::Error>;
- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder <Self::Backend as Backend>::ComputeEncoder);
- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#args ,)* encoder: &'encoder <Self::Backend as Backend>::ComputeEncoder, predicate: Option<impl BufferArg<'predicate, <Self::Backend as Backend>::NativeBuffer>>);
+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder);
+ fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder, predicate: Option<impl BufferArg<'predicate, <Self::Backend as Backend>::NativeBuffer>>);
}
};
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rs
index feee0dd60..778391367 100644
--- a/crates/uzu/build/cpu/compiler.rs
+++ b/crates/uzu/build/cpu/compiler.rs
@@ -431,7 +431,7 @@ impl CpuCompiler {
});
quote! {
- encoder.borrow_mut().push_command(move || {
+ encoder.push_command(move || {
#monomorphized_function(#function_call_args_joined)
});
}
@@ -524,11 +524,11 @@ impl CpuCompiler {
})
}
- fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>) {
+ fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer) {
self.encode_if(#(#encode_args_passthrough ,)* encoder, None::<&Box<[u8]>>)
}
- fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {
+ fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {
#(#argument_copies)*
#encode_body
}
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index 76060401e..302655fd7 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -320,13 +320,13 @@ pub fn bindgen(
Ok(Self { pipeline #(, #conditional_buffer_sets)* })
}
- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {
+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {
#empty_dispatch_guards
compute_encoder.set_compute_pipeline_state(&self.pipeline);
#(#encode_args_sets)*
#dispatch
}
- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {
+ fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {
#empty_dispatch_guards
if let Some(predicate) = predicate {
@@ -334,10 +334,10 @@ pub fn bindgen(
compute_encoder.condition(
__dsl_buffer,
__dsl_offset,
- || {
+ |compute_encoder| {
self.encode(#(#encode_args_names, )* compute_encoder);
},
- None::<fn()>,
+ None::<fn(&mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>)>,
);
} else {
self.encode(#(#encode_args_names, )* compute_encoder);
diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rs
index 76fd12630..ba6542b80 100644
--- a/crates/uzu/src/backends/common/kernel/attention.rs
+++ b/crates/uzu/src/backends/common/kernel/attention.rs
@@ -49,7 +49,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
pub fn encode(
&self,
context: &B::Context,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
args: &AttentionGemmArguments<B>,
) -> Result<(), B::Error> {
let bk: usize = if args.head_dim < 128 {
@@ -138,7 +138,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
args.sinks_buffer,
args.num_heads as u32,
args.suffix_length as u32,
- &compute_encoder,
+ compute_encoder,
);
Ok(())
diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rs
index 174dd259c..dbe4e942f 100644
--- a/crates/uzu/src/backends/common/kernel/audio.rs
+++ b/crates/uzu/src/backends/common/kernel/audio.rs
@@ -120,68 +120,68 @@ pub struct AudioClampArguments<'a, B: Backend> {
pub trait AudioKernelRuntime<B: Backend> {
fn encode_fsq_decode(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioFsqDecodeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_fsq_encode(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioFsqEncodeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_conv1d(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioConv1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_causal_conv1d(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioCausalConv1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_causal_conv_transpose1d(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioCausalConvTranspose1dArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_half_snake(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioHalfSnakeArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_leaky_relu(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioElementwiseArguments<'_, B>,
negative_slope: f32,
) -> Result<(), B::Error>;
fn encode_tanh(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioElementwiseArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_add(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioAddArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_scale(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioScaleArguments<'_, B>,
) -> Result<(), B::Error>;
fn encode_clamp(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
arguments: AudioClampArguments<'_, B>,
) -> Result<(), B::Error>;
}
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
index 839b637ea..78860d0e3 100644
--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -70,7 +70,7 @@ impl<B: Backend> KVCacheUpdate<B> {
in_place_data: &[KVLayerData<B>],
source_indices: &[usize],
destination_indices: &[usize],
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) -> Result<(), KVCacheUpdateError<B>> {
if source_indices.len() != destination_indices.len() {
return Err(KVCacheUpdateError::IndicesCountMismatch);
diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
index 22476608f..8d5423b4d 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
@@ -23,7 +23,7 @@ pub trait FullPrecisionMatmulKernel: Sized {
fn encode(
&mut self,
context: &<Self::Backend as Backend>::Context,
- encoder: &<Self::Backend as Backend>::ComputeEncoder,
+ encoder: &mut <Self::Backend as Backend>::ComputeEncoder,
arguments: FullPrecisionMatmulArguments<Self::Backend>,
);
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
index 0d4b30556..095ce436a 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
@@ -65,7 +65,7 @@ where
context: &B::Context,
arguments: &MatmulArguments<B>,
dispatch_descriptor: &DispatchDescriptor,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) -> Result<(), B::Error> {
let config = dispatch_descriptor.specialization;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
index 950fc9980..5bee72487 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs
@@ -66,7 +66,7 @@ where
context: &B::Context,
arguments: &MatmulArguments<B>,
dispatch_descriptor: &DispatchDescriptor,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) -> Result<(), B::Error> {
let config = dispatch_descriptor.specialization;
let pipeline = self.get_or_create_kernel(context, config)?;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
index 2473e601a..a3f3588f0 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs
@@ -75,7 +75,7 @@ where
context: &B::Context,
arguments: &MatmulArguments<B>,
dispatch_descriptor: &MatmulDispatchDescriptor,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) -> Result<(), B::Error> {
match dispatch_descriptor {
MatmulDispatchDescriptor::Gemv(d) => {
@@ -98,7 +98,7 @@ where
context: &B::Context,
arguments: MatmulArguments<B>,
dispatch_descriptor: &MatmulDispatchDescriptor,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) -> Result<(), B::Error> {
self.encode_dispatch_descriptor(context, &arguments, dispatch_descriptor, encoder)?;
@@ -116,7 +116,7 @@ where
context: &B::Context,
arguments: &MatmulArguments<B>,
bias: &B::NativeBuffer,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) -> Result<(), B::Error> {
let m = arguments.batch as usize;
let n = arguments.output_dim as usize;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
index 5506798bb..91f50eb46 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs
@@ -63,7 +63,7 @@ where
context: &B::Context,
arguments: &MatmulArguments<B>,
dispatch_descriptor: &DispatchDescriptor,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) -> Result<(), B::Error> {
self.ensure_kernels(context)?;
self.ensure_accumulator_buffer(context, dispatch_descriptor.accumulator_bytes)?;
diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
index 7139b277c..1f28e0aed 100644
--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -27,7 +27,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
pub fn encode(
&self,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
fused_up: &B::NativeBuffer,
hidden: &B::NativeBuffer,
m: i32,
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
index 70ca68396..e59cebd61 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
@@ -109,7 +109,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
args.gate_clip_max,
args.up_clip_min,
args.up_clip_max,
- &encoder,
+ encoder,
);
});
@@ -126,7 +126,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
args.d_model as u32,
args.d_ff as u32,
args.k as u32,
- &encoder,
+ encoder,
);
});
}
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index 44eeb1fba..305749f35 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -161,7 +161,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
pub fn encode(
&self,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
arguments: QuantizedMatmulArguments<B>,
) -> Result<(), QuantizedMatmulError<B>> {
if arguments.quantization_type != self.quantization_type {
diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rs
index eb62dc628..7f45c5196 100644
--- a/crates/uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/uzu/src/backends/common/kernel/sampling.rs
@@ -144,7 +144,7 @@ impl<B: Backend> SamplingKernel<B> {
sampling_method: SamplingMethod,
batch_size: usize,
vocab_size: usize,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
) -> Result<(), SamplingError<B>> {
if batch_size > self.max_batch_size {
return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));
diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
index b65adaa73..e478ba466 100644
--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs
@@ -55,7 +55,7 @@ impl<B: Backend> SSDPrefillKernels<B> {
pub fn encode(
&self,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
args: SSDPrefillArguments<B>,
mode: SSDPrefillMode,
) {
diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rs
index ac7bd8408..6ac60bea7 100644
--- a/crates/uzu/src/backends/cpu/backend.rs
+++ b/crates/uzu/src/backends/cpu/backend.rs
@@ -9,9 +9,9 @@ pub struct Cpu;
impl Backend for Cpu {
type Context = CpuContext;
type NativeBuffer = Box<[u8]>;
- type CommandBuffer = RefCell<CpuCommandBuffer>;
- type ComputeEncoder = RefCell<CpuCommandBuffer>;
- type CopyEncoder = RefCell<CpuCommandBuffer>;
+ type CommandBuffer = CpuCommandBuffer;
+ type ComputeEncoder = CpuCommandBuffer;
+ type CopyEncoder = CpuCommandBuffer;
type Event = RefCell<u64>;
type Kernels = CpuKernels;
type Error = CpuError;
diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rs
index fa780d1e2..c627de861 100644
--- a/crates/uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu/src/backends/cpu/command_buffer.rs
@@ -1,4 +1,7 @@
-use std::{cell::RefCell, time::Instant};
+use std::{
+ cell::{OnceCell, RefCell},
+ time::Instant,
+};
use super::Cpu;
use crate::backends::common::{CommandBuffer, CopyEncoder};
@@ -6,16 +9,16 @@ use crate::backends::common::{CommandBuffer, CopyEncoder};
pub struct CpuCommandBuffer {
commands: Vec<Box<dyn Fn()>>,
completion_handlers: Vec<Box<dyn Fn()>>,
- gpu_execution_time_ms: Option<f64>,
+ gpu_execution_time_ms: OnceCell<f64>,
}
impl CpuCommandBuffer {
- pub fn new() -> RefCell<CpuCommandBuffer> {
- RefCell::new(CpuCommandBuffer {
+ pub fn new() -> CpuCommandBuffer {
+ CpuCommandBuffer {
commands: Vec::new(),
completion_handlers: Vec::new(),
- gpu_execution_time_ms: None,
- })
+ gpu_execution_time_ms: OnceCell::new(),
+ }
}
pub fn push_command(
@@ -26,19 +29,19 @@ impl CpuCommandBuffer {
}
}
-impl CommandBuffer for RefCell<CpuCommandBuffer> {
+impl CommandBuffer for CpuCommandBuffer {
type Backend = Cpu;
fn with_compute_encoder<T>(
&mut self,
- callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,
+ callback: impl FnOnce(&mut CpuCommandBuffer) -> T,
) -> T {
callback(self)
}
fn with_copy_encoder<T>(
&mut self,
- callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,
+ callback: impl FnOnce(&mut CpuCommandBuffer) -> T,
) -> T {
callback(self)
}
@@ -49,7 +52,7 @@ impl CommandBuffer for RefCell<CpuCommandBuffer> {
value: u64,
) {
let event = event.as_ptr();
- self.borrow_mut().push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))
+ self.push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))
}
fn encode_signal_event(
@@ -58,23 +61,25 @@ impl CommandBuffer for RefCell<CpuCommandBuffer> {
value: u64,
) {
let event = event.as_ptr();
- self.borrow_mut().push_command(Box::new(move || unsafe { *event = value }))
+ self.push_command(Box::new(move || unsafe { *event = value }))
}
fn add_completion_handler(
&mut self,
handler: impl Fn() + 'static,
) {
- self.borrow_mut().completion_handlers.push(Box::new(move || handler()))
+ self.completion_handlers.push(Box::new(move || handler()))
}
fn submit(&self) {
let start = Instant::now();
- for command in &self.borrow().commands {
+ for command in &self.commands {
command()
}
- self.borrow_mut().gpu_execution_time_ms = Some((Instant::now() - start).as_secs_f64() * 1000.0);
- for completion_handler in self.borrow().completion_handlers.iter() {
+ self.gpu_execution_time_ms
+ .set((Instant::now() - start).as_secs_f64() * 1000.0)
+ .expect("gpu execution time already set");
+ for completion_handler in self.completion_handlers.iter() {
completion_handler();
}
}
@@ -82,11 +87,11 @@ impl CommandBuffer for RefCell<CpuCommandBuffer> {
fn wait_until_completed(&self) {}
fn gpu_execution_time_ms(&self) -> Option<f64> {
- self.borrow().gpu_execution_time_ms
+ self.gpu_execution_time_ms.get().copied()
}
}
-impl CopyEncoder for RefCell<CpuCommandBuffer> {
+impl CopyEncoder for CpuCommandBuffer {
type Backend = Cpu;
fn encode_copy(
@@ -97,7 +102,7 @@ impl CopyEncoder for RefCell<CpuCommandBuffer> {
) {
let src = src.as_ptr();
let dst = dst.as_ptr() as *mut u8;
- self.borrow_mut().push_command(Box::new(move || unsafe {
+ self.push_command(Box::new(move || unsafe {
std::ptr::copy(src, dst, size);
}))
}
@@ -110,7 +115,7 @@ impl CopyEncoder for RefCell<CpuCommandBuffer> {
) {
let size = range.end - range.start;
let dst = dst[range].as_ptr() as *mut u8;
- self.borrow_mut().push_command(Box::new(move || unsafe {
+ self.push_command(Box::new(move || unsafe {
dst.write_bytes(value, size);
}))
}
diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rs
index cbf2d0aa2..4efd26ec3 100644
--- a/crates/uzu/src/backends/cpu/context.rs
+++ b/crates/uzu/src/backends/cpu/context.rs
@@ -31,7 +31,7 @@ impl Context for CpuContext {
Ok(vec![0; size].into_boxed_slice())
}
- fn create_command_buffer(&self) -> Result<RefCell<CpuCommandBuffer>, CpuError> {
+ fn create_command_buffer(&self) -> Result<CpuCommandBuffer, CpuError> {
Ok(CpuCommandBuffer::new())
}
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
index ec3bb5c04..406210962 100644
--- a/crates/uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -27,14 +27,10 @@ include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));
use crate::{
DataType,
backends::{
- common::{
- Backend,
- kernel::matmul::{
- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,
- MatmulKernels,
- },
+ common::kernel::matmul::{
+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulKernels,
},
- cpu::{Cpu, context::CpuContext, error::CpuError},
+ cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError},
},
};
@@ -53,10 +49,10 @@ impl FullPrecisionMatmulKernelTrait for FullPrecisionMatmulCpuKernel {
fn encode(
&mut self,
#[allow(unused)] context: &CpuContext,
- #[allow(unused)] encoder: &<Cpu as Backend>::ComputeEncoder,
+ #[allow(unused)] encoder: &mut CpuCommandBuffer,
#[allow(unused)] arguments: FullPrecisionMatmulArguments<Cpu>,
) {
- encoder.borrow_mut().push_command(move || todo!());
+ encoder.push_command(move || todo!());
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index b13b6a219..33395af71 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -29,7 +29,7 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {
fn encode(
&mut self,
context: &MetalContext,
- encoder: &<Metal as crate::backends::common::Backend>::ComputeEncoder,
+ encoder: &mut <Metal as crate::backends::common::Backend>::ComputeEncoder,
arguments: FullPrecisionMatmulArguments<Metal>,
) {
let mut matmul_arguments = MatmulArguments {
diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
index b231b3219..9c6e5cf95 100644
--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs
@@ -20,17 +20,17 @@ trait ComputeEncoderRawConditional {
/// Safe conditional wrapper that uses the raw unsafe methods internally.
pub trait ComputeEncoderConditional {
fn condition<IfBlock, ElseBlock>(
- &self,
+ &mut self,
predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,
offset: usize,
if_block: IfBlock,
else_block: Option<ElseBlock>,
) where
- IfBlock: FnOnce(),
- ElseBlock: FnOnce();
+ IfBlock: FnOnce(&mut Self),
+ ElseBlock: FnOnce(&mut Self);
}
-impl ComputeEncoderRawConditional for ProtocolObject<dyn MTLComputeCommandEncoder> {
+impl ComputeEncoderRawConditional for Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {
fn encode_start_if(
&self,
predicate: &ProtocolObject<dyn MTLBuffer>,
@@ -64,20 +64,20 @@ where
T: ComputeEncoderRawConditional,
{
fn condition<IfBlock, ElseBlock>(
- &self,
+ &mut self,
predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,
offset: usize,
if_block: IfBlock,
else_block: Option<ElseBlock>,
) where
- IfBlock: FnOnce(),
- ElseBlock: FnOnce(),
+ IfBlock: FnOnce(&mut Self),
+ ElseBlock: FnOnce(&mut Self),
{
self.encode_start_if(predicate, offset, MTLCompareFunction::Equal, 0);
- if_block();
+ if_block(self);
if let Some(else_block) = else_block {
self.encode_start_else();
- else_block();
+ else_block(self);
}
self.encode_end_if();
}
diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rs
index 133866eaa..59e0bc5bf 100644
--- a/crates/uzu/src/encodable_block/activation.rs
+++ b/crates/uzu/src/encodable_block/activation.rs
@@ -47,7 +47,7 @@ impl<B: Backend> EncodableBlock<B> for Activation<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let input_array = arrays[0].borrow_mut();
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index 444127f32..5a6412e4a 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -134,7 +134,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
parameters: &EncodingParameters<B>,
command_buffer: &mut B::CommandBuffer,
) {
- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, &encoder));
+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
if parameters.wait_until_completed {
command_buffer.submit();
@@ -150,7 +150,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- compute_encoder: &B::ComputeEncoder,
+ compute_encoder: &mut B::ComputeEncoder,
) {
let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
@@ -260,7 +260,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
suffix_length as u32,
0u32,
max_sequence_length as u32,
- &compute_encoder,
+ compute_encoder,
);
}
@@ -331,7 +331,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
suffix_length as u32,
segment_prefix_length as u32,
max_sequence_length as u32,
- &compute_encoder,
+ compute_encoder,
);
}
@@ -401,7 +401,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
sinks_buffer,
num_heads as u32,
suffix_length as u32,
- &compute_encoder,
+ compute_encoder,
)
},
KernelVariant::TwoPass => {
@@ -434,7 +434,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
mask_q_seq_stride_opt,
mask_head_stride_opt,
sinks_buffer,
- &compute_encoder,
+ compute_encoder,
);
kernel_pass2.encode(
partials_buf_borrow.deref(),
@@ -443,7 +443,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
attention_output_buf_borrow.deref(),
num_heads as u32,
suffix_length as u32,
- &compute_encoder,
+ compute_encoder,
);
},
}
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index e2389ee61..c1c1b8a8d 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -382,7 +382,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
debug_assert!(
self.supports_shared_encoder(),
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index ac62d59fc..39adbb7eb 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -166,7 +166,7 @@ impl<B: Backend> EncodableBlock<B> for Decoder<B> {
&self,
_state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- _encoder: &B::ComputeEncoder,
+ _encoder: &mut B::ComputeEncoder,
) {
unimplemented!("Decoder does not support shared encoder")
}
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
index 910273e7e..95d9a79d8 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
@@ -95,7 +95,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
index 5342b58d3..d55aada5d 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
@@ -99,7 +99,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
let batch_size = state.sampling_length();
diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rs
index 6b748773a..237b7f402 100644
--- a/crates/uzu/src/encodable_block/full_precision_linear.rs
+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs
@@ -131,7 +131,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index f49b3dc76..af8d857e4 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -429,7 +429,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
debug_assert!(
self.supports_shared_encoder(),
diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rs
index 3992419bb..47376d891 100644
--- a/crates/uzu/src/encodable_block/layer_norm.rs
+++ b/crates/uzu/src/encodable_block/layer_norm.rs
@@ -74,7 +74,7 @@ impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index f9e0c1fc2..1746a3152 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -138,7 +138,7 @@ impl<B: Backend> MambaMixer<B> {
fn encode_pipeline_with_encoder(
&self,
state: &mut ForwardPassState<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
parameters: &EncodingParameters<B>,
) {
let active_suffix_length = state.active_suffix_length();
@@ -162,7 +162,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_split_inproj(
&self,
state: &mut ForwardPassState<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -202,7 +202,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_conv_scan(
&self,
state: &mut ForwardPassState<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -246,7 +246,7 @@ impl<B: Backend> MambaMixer<B> {
suffix_length as u32,
inner_dim as u32,
proj_dim as u32,
- &encoder,
+ encoder,
);
}
} else {
@@ -262,7 +262,7 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
suffix_length as u32,
conv_dim as u32,
- &encoder,
+ encoder,
);
Some(buffer)
@@ -290,7 +290,7 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
inner_dim as u32,
proj_dim as u32,
- &encoder,
+ encoder,
)
}
}
@@ -299,7 +299,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_prefill_ssm(
&self,
state: &mut ForwardPassState<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
suffix_length: usize,
) {
let base_arrays = state.arrays(&[
@@ -350,7 +350,7 @@ impl<B: Backend> MambaMixer<B> {
fn run_decode_ssm(
&self,
state: &mut ForwardPassState<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -481,7 +481,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
self.encode_pipeline_with_encoder(state, encoder, parameters);
}
diff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rs
index 17bf482de..3da5af103 100644
--- a/crates/uzu/src/encodable_block/mlp.rs
+++ b/crates/uzu/src/encodable_block/mlp.rs
@@ -75,7 +75,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
&self,
state: &mut ForwardPassState<B>,
params: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
// Up
self.up.encode_with_shared_encoder(state, params, encoder);
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index d1ae53711..2f1abc358 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -107,7 +107,7 @@ pub trait EncodableBlock<B: Backend> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
);
fn supports_shared_encoder(&self) -> bool {
diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rs
index 61a4636ae..56bd2b891 100644
--- a/crates/uzu/src/encodable_block/moe_block.rs
+++ b/crates/uzu/src/encodable_block/moe_block.rs
@@ -308,7 +308,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
e as u32,
k as u32,
self.router_renorm,
- &encoder,
+ encoder,
);
});
}
@@ -322,7 +322,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
suffix_length as u32,
e as u32,
k as u32,
- &encoder,
+ encoder,
);
});
@@ -337,7 +337,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
num_blocks as u32,
num_tiles as u32,
0u32,
- &encoder,
+ encoder,
);
});
command_buffer.with_compute_encoder(|encoder| {
@@ -355,7 +355,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
num_blocks as u32,
num_tiles as u32,
tok2row_buf.borrow().deref(),
- &encoder,
+ encoder,
);
});
@@ -476,7 +476,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
suffix_length as u32,
self.model_dim as u32,
k as u32,
- &encoder,
+ encoder,
);
});
@@ -494,7 +494,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
&self,
_state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- _encoder: &B::ComputeEncoder,
+ _encoder: &mut B::ComputeEncoder,
) {
unreachable!("MoeBlock does not support shared compute encoder");
}
diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rs
index b040f39af..0553ecef4 100644
--- a/crates/uzu/src/encodable_block/normalization.rs
+++ b/crates/uzu/src/encodable_block/normalization.rs
@@ -70,7 +70,7 @@ impl<B: Backend> EncodableBlock<B> for Normalization<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
match self {
Self::LayerNorm(layer_norm) => layer_norm.encode_with_shared_encoder(state, parameters, encoder),
diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rs
index 1424c6013..f5c75e496 100644
--- a/crates/uzu/src/encodable_block/pooling.rs
+++ b/crates/uzu/src/encodable_block/pooling.rs
@@ -26,7 +26,7 @@ impl<B: Backend> PoolingKernel<B> {
seq_len: u32,
hidden_dim: u32,
batch_size: u32,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
match self {
Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),
@@ -84,7 +84,7 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let batch_size = 1;
let seq_len = state.aux_buffers_suffix_length();
diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rs
index 6cbca8760..0b5bf96aa 100644
--- a/crates/uzu/src/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/encodable_block/prediction_head.rs
@@ -78,7 +78,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
self.dense.encode_with_shared_encoder(state, parameters, encoder);
self.activation.encode_with_shared_encoder(state, parameters, encoder);
diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
index 4d51f0932..cbadad192 100644
--- a/crates/uzu/src/encodable_block/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -126,7 +126,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
let qkv_shape = {
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
index 7d27262f4..f9675456c 100644
--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
+++ b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
@@ -243,7 +243,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
index 1576d8d8c..b63dd7305 100644
--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
@@ -233,7 +233,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
let batch_size = state.sampling_length();
diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rs
index fa8f4fdd4..da89dd758 100644
--- a/crates/uzu/src/encodable_block/quantized_linear.rs
+++ b/crates/uzu/src/encodable_block/quantized_linear.rs
@@ -248,7 +248,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index a1b7cea4c..433646f55 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -88,7 +88,7 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let input_binding = state.arrays(&[self.input_array_id]);
let output_binding = state.arrays(&[self.output_array_id]);
diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rs
index 2cbd65be8..dc450d574 100644
--- a/crates/uzu/src/encodable_block/rope.rs
+++ b/crates/uzu/src/encodable_block/rope.rs
@@ -39,7 +39,7 @@ impl<B: Backend> EncodableBlock<B> for Rope<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
let qkv_binding = state.arrays(&[ArrayId::QKV]);
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index e3644d572..846cf8b22 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -49,7 +49,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index dccc21457..e4a715d45 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -116,7 +116,7 @@ impl<B: Backend> ShortConvMixer<B> {
self.in_projection.encode(state, parameters, command_buffer);
- command_buffer.with_compute_encoder(|encoder| self.run_conv(state, &encoder, active_suffix_length));
+ command_buffer.with_compute_encoder(|encoder| self.run_conv(state, encoder, active_suffix_length));
self.out_projection.encode(state, parameters, command_buffer);
@@ -129,7 +129,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn encode_pipeline_with_encoder(
&self,
state: &mut ForwardPassState<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
parameters: &EncodingParameters<B>,
) {
let active_suffix_length = state.active_suffix_length();
@@ -173,7 +173,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &B::ComputeEncoder,
+ compute: &mut B::ComputeEncoder,
active_suffix_length: usize,
) {
self.clear_suffix_state_valid_range(state);
@@ -212,7 +212,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_prefill_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &B::ComputeEncoder,
+ compute: &mut B::ComputeEncoder,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -269,7 +269,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_trie_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &B::ComputeEncoder,
+ compute: &mut B::ComputeEncoder,
sampling_start: usize,
trie_len: usize,
) {
@@ -328,7 +328,7 @@ impl<B: Backend> ShortConvMixer<B> {
fn run_decode_conv(
&self,
state: &mut ForwardPassState<B>,
- compute: &B::ComputeEncoder,
+ compute: &mut B::ComputeEncoder,
suffix_length: usize,
) {
if self.model_dim == 0 || suffix_length == 0 {
@@ -374,7 +374,7 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {
) {
if self.supports_shared_encoder() {
command_buffer
- .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, &encoder, parameters));
+ .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));
if parameters.wait_until_completed {
command_buffer.submit();
@@ -393,7 +393,7 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
self.encode_pipeline_with_encoder(state, encoder, parameters);
}
diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rs
index 9aa6ba3da..284794698 100644
--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs
@@ -40,7 +40,7 @@ impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rs
index 211c398a1..1c9312274 100644
--- a/crates/uzu/src/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/encodable_block/tensor_copy.rs
@@ -40,7 +40,7 @@ impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {
&self,
state: &mut ForwardPassState<B>,
_parameters: &EncodingParameters<B>,
- encoder: &B::ComputeEncoder,
+ encoder: &mut B::ComputeEncoder,
) {
let arrays = state.arrays(&self.argument_arrays);
assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index dd27362c8..fad69fc8f 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -553,7 +553,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
if update.unmask_col >= 0 || update.mask_col >= 0 {
let mask_buf_rc = mask_buffer.borrow().buffer();
let mask_buf_borrow = mask_buf_rc.borrow();
- mask_update.encode(mask_buf_borrow.deref(), update.unmask_col, update.mask_col, &encoder);
+ mask_update.encode(mask_buf_borrow.deref(), update.unmask_col, update.mask_col, encoder);
}
}
}
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index 085e0546c..8aa469b44 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -317,7 +317,7 @@ fn run_single_pass_attention(
.expect("Failed to create buffer");
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -347,7 +347,7 @@ fn run_single_pass_attention(
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
@@ -390,7 +390,7 @@ fn run_single_pass_attention_with_is_causal(
.expect("Failed to create buffer");
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -419,7 +419,7 @@ fn run_single_pass_attention_with_is_causal(
sinks_buffer.as_ref().map(|b| b),
num_heads as u32,
seq_len as u32,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
@@ -461,7 +461,7 @@ fn run_gemm_attention(
.expect("Failed to create buffer");
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let args = AttentionGemmArguments {
queries_buffer: &query_buffer,
@@ -481,7 +481,7 @@ fn run_gemm_attention(
scale,
};
- let encode_result = kernel.encode(context, &compute_encoder, &args);
+ let encode_result = kernel.encode(context, &mut compute_encoder, &args);
compute_encoder.end_encoding();
encode_result?;
@@ -936,7 +936,7 @@ fn run_two_pass_attention(
.expect("Failed to create buffer");
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let mut mask_kv_seq_stride: Option<u32> = None;
let mut mask_q_seq_stride: Option<u32> = None;
@@ -967,7 +967,7 @@ fn run_two_pass_attention(
mask_q_seq_stride,
mask_head_stride,
sinks_buffer.as_ref().map(|b| b),
- &compute_encoder,
+ &mut compute_encoder,
);
kernel_pass2.encode(
&partials_buffer,
@@ -976,7 +976,7 @@ fn run_two_pass_attention(
&output_buffer,
num_heads as u32,
seq_len as u32,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
@@ -1174,7 +1174,7 @@ fn perf_two_pass_attention() {
// ---- Launch and time ----
let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;
@@ -1199,7 +1199,7 @@ fn perf_two_pass_attention() {
None,
None,
sinks_buffer.as_ref().map(|b| b),
- &compute_encoder,
+ &mut compute_encoder,
);
kernel_pass2.encode(
&partials_buffer,
@@ -1208,7 +1208,7 @@ fn perf_two_pass_attention() {
&output_buffer,
num_heads as u32,
suffix_length as u32,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rs
index 3075e4dbb..8e5348c2d 100644
--- a/crates/uzu/tests/audio_codec_test.rs
+++ b/crates/uzu/tests/audio_codec_test.rs
@@ -99,7 +99,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {
}
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
kernel.encode(
input.buffer(),
@@ -117,7 +117,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {
padding as i32,
1_i32,
batch_size as i32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
@@ -205,7 +205,7 @@ fn audio_causal_conv1d_matches_reference_f32() {
}
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
kernel.encode(
input.buffer(),
@@ -219,7 +219,7 @@ fn audio_causal_conv1d_matches_reference_f32() {
kernel_size as i32,
dilation as i32,
batch_size as i32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
@@ -320,7 +320,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
}
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
kernel.encode(
input.buffer(),
@@ -335,7 +335,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
stride as i32,
groups as i32,
batch_size as i32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
@@ -370,9 +370,9 @@ fn audio_leaky_relu_matches_reference_f32() {
let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
- kernel.encode(input.buffer(), output.buffer(), n as i32, slope, &encoder);
+ kernel.encode(input.buffer(), output.buffer(), n as i32, slope, &mut encoder);
encoder.end_encoding();
command_buffer.commit();
@@ -405,9 +405,9 @@ fn audio_tanh_matches_reference_f32() {
let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
- kernel.encode(input.buffer(), output.buffer(), n as i32, &encoder);
+ kernel.encode(input.buffer(), output.buffer(), n as i32, &mut encoder);
encoder.end_encoding();
command_buffer.commit();
@@ -444,11 +444,11 @@ fn audio_add_and_scale_match_reference_f32() {
let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
- add_kernel.encode(a.buffer(), b.buffer(), sum.buffer(), n as i32, &encoder);
+ add_kernel.encode(a.buffer(), b.buffer(), sum.buffer(), n as i32, &mut encoder);
- scale_kernel.encode(sum.buffer(), scaled.buffer(), n as i32, scale_value, &encoder);
+ scale_kernel.encode(sum.buffer(), scaled.buffer(), n as i32, scale_value, &mut encoder);
encoder.end_encoding();
command_buffer.commit();
@@ -494,9 +494,9 @@ fn audio_clamp_matches_reference_f32() {
let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
- kernel.encode(input.buffer(), output.buffer(), n as i32, min_value, max_value, &encoder);
+ kernel.encode(input.buffer(), output.buffer(), n as i32, min_value, max_value, &mut encoder);
encoder.end_encoding();
command_buffer.commit();
@@ -537,7 +537,7 @@ fn audio_half_snake_matches_reference_f32() {
let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
kernel.encode(
input.buffer(),
@@ -549,7 +549,7 @@ fn audio_half_snake_matches_reference_f32() {
negative_slope,
eps,
batch_size as i32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
@@ -608,7 +608,7 @@ fn audio_fsq_decode_matches_reference() {
context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
kernel.encode(
tokens.buffer(),
@@ -619,7 +619,7 @@ fn audio_fsq_decode_matches_reference() {
codebook_dim as i32,
&num_levels,
batch_size as i32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
@@ -671,7 +671,7 @@ fn audio_fsq_encode_matches_reference() {
let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");
let command_buffer = context.command_queue.command_buffer().expect("command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
kernel.encode(
input.buffer(),
@@ -684,7 +684,7 @@ fn audio_fsq_encode_matches_reference() {
&dim_base_index,
eps,
batch_size as i32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 4bf11fe20..bb2e70e9c 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -46,7 +46,7 @@ fn run_metal_matmul(
let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
let mut arguments = MatmulArguments {
a: &a_buf,
a_offset: 0,
@@ -64,7 +64,7 @@ fn run_metal_matmul(
};
MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);
let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");
- let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &enc);
+ let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut enc);
enc.end_encoding();
encode_result.expect("encode");
cb.commit();
diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
index 85cb7bb02..78e3aab7e 100644
--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs
+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
@@ -76,7 +76,7 @@ fn gen_topk_ids_from_logits(
// Use fused router+topk kernel
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");
+ let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");
let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16)
.expect("router_topk kernel");
router_topk.encode(
@@ -90,7 +90,7 @@ fn gen_topk_ids_from_logits(
e as u32,
k as u32,
true,
- &router_topk_encoder,
+ &mut router_topk_encoder,
);
router_topk_encoder.end_encoding();
cb.commit();
@@ -127,7 +127,7 @@ fn test_counts_offsets_fused_parity_random() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
kernel.encode(
&topk_ids_buf,
@@ -137,7 +137,7 @@ fn test_counts_offsets_fused_parity_random() {
t as u32,
e as u32,
k as u32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
@@ -179,8 +179,8 @@ fn test_counts_offsets_fused_edge_cases() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
- kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &mut encoder);
encoder.end_encoding();
cb.commit();
cb.wait_until_completed();
@@ -203,8 +203,8 @@ fn test_counts_offsets_fused_edge_cases() {
let kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
- kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &mut encoder);
encoder.end_encoding();
cb.commit();
cb.wait_until_completed();
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index 874a4b28b..fb5fdf845 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -351,7 +351,7 @@ fn run_moe_parity_test_internal(
// Router + TopK (fused kernel)
let router_topk =
<<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk");
- let router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");
+ let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -363,17 +363,26 @@ fn run_moe_parity_test_internal(
e as u32,
k as u32,
true,
- &router_topk_encoder,
+ &mut router_topk_encoder,
);
router_topk_encoder.end_encoding();
let fused_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
- fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
+ fused_kernel.encode(
+ &topk_ids_buf,
+ &offsets_buf,
+ &sumk_buf,
+ &partials_buf,
+ t as u32,
+ e as u32,
+ k as u32,
+ &mut encoder,
+ );
encoder.end_encoding();
- let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
+ let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx)
.expect("scatter bases kernel");
scatter_bases_kernel.encode(
@@ -384,11 +393,11 @@ fn run_moe_parity_test_internal(
num_blocks as u32,
num_tiles as u32,
0u32,
- &scatter_bases_encoder,
+ &mut scatter_bases_encoder,
);
scatter_bases_encoder.end_encoding();
- let scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");
+ let mut scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");
let scatter_map_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16)
.expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");
@@ -406,7 +415,7 @@ fn run_moe_parity_test_internal(
num_blocks as u32,
num_tiles as u32,
&tok2row_buf,
- &scatter_encoder,
+ &mut scatter_encoder,
);
scatter_encoder.end_encoding();
@@ -464,7 +473,7 @@ fn run_moe_parity_test_internal(
let finalize =
<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
&tok2row_buf,
&topk_probs_buf,
@@ -473,7 +482,7 @@ fn run_moe_parity_test_internal(
t as u32,
d_model as u32,
k as u32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rs
index 77254d876..077e7c797 100644
--- a/crates/uzu/tests/moe/moe_finalize_test.rs
+++ b/crates/uzu/tests/moe/moe_finalize_test.rs
@@ -91,7 +91,7 @@ fn test_finalize_correctness() {
let finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16)
.expect("finalize kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
&tok2row_buf,
&probs_buf,
@@ -100,7 +100,7 @@ fn test_finalize_correctness() {
t as u32,
d_model as u32,
k as u32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index f62468bb1..fed88bc9a 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -122,7 +122,7 @@ fn test_moe_e2e_decode_perf() {
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -134,7 +134,7 @@ fn test_moe_e2e_decode_perf() {
e as u32,
k as u32,
true,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
@@ -183,7 +183,7 @@ fn test_moe_e2e_prefill_perf() {
// Time fused Router+TopK
let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
router_topk.encode(
&x_buf,
&router_w_buf,
@@ -195,7 +195,7 @@ fn test_moe_e2e_prefill_perf() {
e as u32,
k as u32,
true,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
@@ -312,7 +312,7 @@ fn test_moe_pipeline_breakdown_decode() {
// Testing: Router + TopK + Counts+Offsets (FUSED)
let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
router_topk_fused_kernel.encode(
&x_buf,
&router_w_buf,
@@ -324,7 +324,7 @@ fn test_moe_pipeline_breakdown_decode() {
e as u32,
k as u32,
true,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
@@ -333,7 +333,7 @@ fn test_moe_pipeline_breakdown_decode() {
let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
counts_offsets_kernel.encode(
&topk_ids_buf,
&offsets_buf,
@@ -342,7 +342,7 @@ fn test_moe_pipeline_breakdown_decode() {
t as u32,
e as u32,
k as u32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
@@ -351,7 +351,7 @@ fn test_moe_pipeline_breakdown_decode() {
let scatter_perf = time_kernel("Scatter", 2, 5, || {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
+ let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");
scatter_bases_kernel.encode(
&partials_buf,
&block_bases_buf,
@@ -360,11 +360,11 @@ fn test_moe_pipeline_breakdown_decode() {
num_blocks as u32,
num_tiles as u32,
0u32,
- &scatter_bases_encoder,
+ &mut scatter_bases_encoder,
);
scatter_bases_encoder.end_encoding();
- let scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");
+ let mut scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");
scatter_map_kernel.encode(
&topk_ids_buf,
&topk_probs_buf,
@@ -379,7 +379,7 @@ fn test_moe_pipeline_breakdown_decode() {
num_blocks as u32,
num_tiles as u32,
&tok2row_buf,
- &scatter_map_encoder,
+ &mut scatter_map_encoder,
);
scatter_map_encoder.end_encoding();
@@ -445,7 +445,7 @@ fn test_moe_pipeline_breakdown_decode() {
let finalize_perf = time_kernel("Finalize", 2, 5, || {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
finalize_kernel.encode(
&tok2row_buf,
&topk_probs_buf,
@@ -454,7 +454,7 @@ fn test_moe_pipeline_breakdown_decode() {
t as u32,
d_model as u32,
k as u32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rs
index d5145c00c..9932b651e 100644
--- a/crates/uzu/tests/moe/moe_router_topk_test.rs
+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs
@@ -158,7 +158,7 @@ fn run_router_topk_once(
let probs_buf = alloc_buffer::<bf16>(ctx, t * k);
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("Failed to create command encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("Failed to create command encoder");
kernel.encode(
&input_buf,
&weight_buf,
@@ -170,7 +170,7 @@ fn run_router_topk_once(
e as u32,
k as u32,
renorm,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rs
index f47893c00..a369da7af 100644
--- a/crates/uzu/tests/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/moe/moe_scatter_test.rs
@@ -85,7 +85,7 @@ fn test_scatter_buckets_parity() {
let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16)
.expect("router_topk");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");
router_topk.encode(
&input_buf,
&weight_buf,
@@ -97,7 +97,7 @@ fn test_scatter_buckets_parity() {
e as u32,
k as u32,
true,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
@@ -117,7 +117,7 @@ fn test_scatter_buckets_parity() {
let fused_kernel =
<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb.new_compute_command_encoder().expect("encoder");
+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");
fused_kernel.encode(
&topk_ids_buf,
&offsets_buf,
@@ -126,7 +126,7 @@ fn test_scatter_buckets_parity() {
t as u32,
e as u32,
k as u32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
cb.commit();
@@ -146,7 +146,8 @@ fn test_scatter_buckets_parity() {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);
- let scatter_bases_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");
+ let mut scatter_bases_encoder =
+ cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");
scatter_bases_kernel.encode(
&partials_buf,
&block_bases_buf,
@@ -155,7 +156,7 @@ fn test_scatter_buckets_parity() {
num_blocks as u32,
num_tiles as u32,
0u32,
- &scatter_bases_encoder,
+ &mut scatter_bases_encoder,
);
scatter_bases_encoder.end_encoding();
@@ -167,7 +168,7 @@ fn test_scatter_buckets_parity() {
let out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");
+ let mut scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");
scatter_kernel.encode(
&topk_ids_buf,
&topk_probs_buf,
@@ -181,7 +182,7 @@ fn test_scatter_buckets_parity() {
k as u32,
num_blocks as u32,
num_tiles as u32,
- &scatter_encoder,
+ &mut scatter_encoder,
);
scatter_encoder.end_encoding();
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index 17d4e7d40..69c25999c 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -511,8 +511,8 @@ fn execute_quantized_matmul(
quantization_type,
};
let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&encoder, args).unwrap();
+ let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&mut encoder, args).unwrap();
encoder.end_encoding();
cb_ref.commit();
cb_ref.wait_until_completed();
@@ -534,8 +534,8 @@ fn execute_quantized_matmul(
quantization_type,
};
let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&encoder, args).unwrap();
+ let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&mut encoder, args).unwrap();
encoder.end_encoding();
cb_ref.commit();
cb_ref.wait_until_completed();
diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rs
index 4f9b430f9..f44a409a8 100644
--- a/crates/uzu/tests/rms_norm_kernel_test.rs
+++ b/crates/uzu/tests/rms_norm_kernel_test.rs
@@ -311,7 +311,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
// Create command buffer and encode
let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(
&input_buffer,
@@ -322,7 +322,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(
epsilon,
0.0,
false,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
@@ -428,7 +428,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(
&input_buffer,
@@ -439,7 +439,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(
epsilon,
0.0,
false,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
@@ -635,7 +635,7 @@ fn perf_rms_norm_with_size(
// ---- Launch and time ----
let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(
&input_buffer,
@@ -646,7 +646,7 @@ fn perf_rms_norm_with_size(
EPSILON,
0.0,
false,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
@@ -802,7 +802,8 @@ fn qk_norm_test() {
// Test Q head normalization
{
let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
q_kernel.encode(
&qkv_buffer,
@@ -817,7 +818,7 @@ fn qk_norm_test() {
0,
num_q_heads as u32,
false,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
@@ -864,7 +865,8 @@ fn qk_norm_test() {
// Test K head normalization
{
let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
k_kernel.encode(
&qkv_buffer,
@@ -879,7 +881,7 @@ fn qk_norm_test() {
num_q_heads as u32,
num_kv_heads as u32,
false,
- &compute_encoder,
+ &mut compute_encoder,
);
compute_encoder.end_encoding();
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index 231fd027f..6744ffcc6 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -111,7 +111,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
let command_buffer = command_buffer_ref.to_owned();
// Run sampling
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
&logits_buffer,
@@ -123,7 +123,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
SamplingMethod::Greedy,
batch_size,
vocab_size,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("Argmax sampling should succeed");
compute_encoder.end_encoding();
@@ -240,7 +240,7 @@ fn test_topp_sampling_from_prob_exact_match(
let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
&logits_buf,
@@ -257,7 +257,7 @@ fn test_topp_sampling_from_prob_exact_match(
},
batch_size,
vocab_size,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("encode");
compute_encoder.end_encoding();
@@ -363,7 +363,7 @@ fn test_topp_sampling_statistical_large() {
let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -381,7 +381,7 @@ fn test_topp_sampling_statistical_large() {
},
BATCH,
VOCAB,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("encode");
compute_encoder.end_encoding();
@@ -465,7 +465,7 @@ fn perf_topp_128k_vocab() {
// ---- Launch once and time ----
let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -483,7 +483,7 @@ fn perf_topp_128k_vocab() {
},
BATCH,
VOCAB,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("encode");
compute_encoder.end_encoding();
@@ -569,7 +569,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
// ---- Launch once and time ----
let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -582,7 +582,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
SamplingMethod::Greedy,
BATCH,
VOCAB,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("encode");
compute_encoder.end_encoding();
@@ -688,7 +688,8 @@ fn test_categorical_sampling() {
let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -706,7 +707,7 @@ fn test_categorical_sampling() {
},
batch_size,
vocab_size,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("Categorical sampling should succeed");
compute_encoder.end_encoding();
@@ -829,7 +830,8 @@ fn test_categorical_sampling_statistical() {
let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder =
+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -847,7 +849,7 @@ fn test_categorical_sampling_statistical() {
},
BATCH,
VOCAB,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("encode");
compute_encoder.end_encoding();
@@ -935,7 +937,7 @@ fn perf_categorical_128k_vocab() {
// Launch and time
let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
@@ -953,7 +955,7 @@ fn perf_categorical_128k_vocab() {
},
BATCH,
VOCAB,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("encode");
compute_encoder.end_encoding();
@@ -1024,8 +1026,8 @@ fn test_temperature_gpu_cpu_match() {
let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &compute_encoder);
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &mut compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
@@ -1090,8 +1092,8 @@ fn test_topk_gpu_cpu_match() {
let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &compute_encoder);
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
@@ -1155,8 +1157,8 @@ fn test_topp_gpu_cpu_match() {
let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &compute_encoder);
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
@@ -1225,8 +1227,8 @@ fn test_minp_gpu_cpu_match() {
let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, MINP, &compute_encoder);
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut compute_encoder);
compute_encoder.end_encoding();
command_buffer_ref.commit();
@@ -1313,7 +1315,7 @@ fn test_minp_sampling_exact_match(
let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel
.encode_with_encoder(
&logits_buf,
@@ -1330,7 +1332,7 @@ fn test_minp_sampling_exact_match(
},
batch_size,
vocab_size,
- &compute_encoder,
+ &mut compute_encoder,
)
.expect("encode");
compute_encoder.end_encoding();
@@ -1412,8 +1414,15 @@ fn test_gumbel_gpu_cpu_match() {
let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");
let command_buffer = command_buffer_ref.to_owned();
- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&logits_buffer, &seeds_buffer, &gumbel_logits_buffer, BATCH as u32, VOCAB as u32, &compute_encoder);
+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(
+ &logits_buffer,
+ &seeds_buffer,
+ &gumbel_logits_buffer,
+ BATCH as u32,
+ VOCAB as u32,
+ &mut compute_encoder,
+ );
compute_encoder.end_encoding();
command_buffer_ref.commit();
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index e33fbd8da..c0e4ffb54 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -249,8 +249,8 @@ fn run_prefill_kernel_mode(
};
let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- kernel.encode(&encoder, args, mode);
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ kernel.encode(&mut encoder, args, mode);
encoder.end_encoding();
command_buffer.commit();
command_buffer.wait_until_completed();
@@ -321,7 +321,7 @@ fn run_conv_scan_once(
}
let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(
&padded_buf,
&w_buf,
@@ -337,7 +337,7 @@ fn run_conv_scan_once(
channels as u32,
channels as u32,
0u32,
- &encoder,
+ &mut encoder,
);
encoder.end_encoding();
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index e952f8eb5..0552a5348 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -241,7 +241,7 @@ fn ssd_update_with_z_bf16() {
let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16).unwrap();
let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
+ let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
kernel.encode(
&x_buf,
&dt_buf,
@@ -261,7 +261,7 @@ fn ssd_update_with_z_bf16() {
bsz as u32,
h as u32,
dh as u32,
- &enc,
+ &mut enc,
);
enc.end_encoding();
cb_ref.commit();