A high-performance inference engine for AI models
more muts (#229)
uuuvn98fd54b197bb
diff --git a/crates/uzu/build/common/traitgen.rs b/crates/uzu/build/common/traitgen.rsindex 537c7f9c4..b7d52f3d5 100644--- a/crates/uzu/build/common/traitgen.rs+++ b/crates/uzu/build/common/traitgen.rs@@ -69,8 +69,8 @@ pub fn traitgen(kernel: &Kernel) -> (TokenStream, TokenStream) {fn new(context: &<Self::Backend as Backend>::Context #(, #params)*) -> Result<Self, <Self::Backend as Backend>::Error>;- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder <Self::Backend as Backend>::ComputeEncoder);- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#args ,)* encoder: &'encoder <Self::Backend as Backend>::ComputeEncoder, predicate: Option<impl BufferArg<'predicate, <Self::Backend as Backend>::NativeBuffer>>);+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder);+ fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#args ,)* encoder: &'encoder mut <Self::Backend as Backend>::ComputeEncoder, predicate: Option<impl BufferArg<'predicate, <Self::Backend as Backend>::NativeBuffer>>);}};diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rsindex feee0dd60..778391367 100644--- a/crates/uzu/build/cpu/compiler.rs+++ b/crates/uzu/build/cpu/compiler.rs@@ -431,7 +431,7 @@ impl CpuCompiler {});quote! {- encoder.borrow_mut().push_command(move || {+ encoder.push_command(move || {#monomorphized_function(#function_call_args_joined)});}@@ -524,11 +524,11 @@ impl CpuCompiler {})}- fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>) {+ fn encode<#(#encode_generics ,)* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer) {self.encode_if(#(#encode_args_passthrough ,)* encoder, None::<&Box<[u8]>>)}- fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder std::cell::RefCell<crate::backends::cpu::command_buffer::CpuCommandBuffer>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {+ fn encode_if<#(#encode_generics ,)* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::cpu::command_buffer::CpuCommandBuffer, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Box<[u8]>>>) {#(#argument_copies)*#encode_body}diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex 76060401e..302655fd7 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -320,13 +320,13 @@ pub fn bindgen(Ok(Self { pipeline #(, #conditional_buffer_sets)* })}- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {#empty_dispatch_guardscompute_encoder.set_compute_pipeline_state(&self.pipeline);#(#encode_args_sets)*#dispatch}- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {+ fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {#empty_dispatch_guardsif let Some(predicate) = predicate {@@ -334,10 +334,10 @@ pub fn bindgen(compute_encoder.condition(__dsl_buffer,__dsl_offset,- || {+ |compute_encoder| {self.encode(#(#encode_args_names, )* compute_encoder);},- None::<fn()>,+ None::<fn(&mut Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>)>,);} else {self.encode(#(#encode_args_names, )* compute_encoder);diff --git a/crates/uzu/src/backends/common/kernel/attention.rs b/crates/uzu/src/backends/common/kernel/attention.rsindex 76fd12630..ba6542b80 100644--- a/crates/uzu/src/backends/common/kernel/attention.rs+++ b/crates/uzu/src/backends/common/kernel/attention.rs@@ -49,7 +49,7 @@ impl<B: Backend> AttentionGemmBlock<B> {pub fn encode(&self,context: &B::Context,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,args: &AttentionGemmArguments<B>,) -> Result<(), B::Error> {let bk: usize = if args.head_dim < 128 {@@ -138,7 +138,7 @@ impl<B: Backend> AttentionGemmBlock<B> {args.sinks_buffer,args.num_heads as u32,args.suffix_length as u32,- &compute_encoder,+ compute_encoder,);Ok(())diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rsindex 174dd259c..dbe4e942f 100644--- a/crates/uzu/src/backends/common/kernel/audio.rs+++ b/crates/uzu/src/backends/common/kernel/audio.rs@@ -120,68 +120,68 @@ pub struct AudioClampArguments<'a, B: Backend> {pub trait AudioKernelRuntime<B: Backend> {fn encode_fsq_decode(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioFsqDecodeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_fsq_encode(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioFsqEncodeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_conv1d(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioConv1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_causal_conv1d(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioCausalConv1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_causal_conv_transpose1d(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioCausalConvTranspose1dArguments<'_, B>,) -> Result<(), B::Error>;fn encode_half_snake(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioHalfSnakeArguments<'_, B>,) -> Result<(), B::Error>;fn encode_leaky_relu(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioElementwiseArguments<'_, B>,negative_slope: f32,) -> Result<(), B::Error>;fn encode_tanh(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioElementwiseArguments<'_, B>,) -> Result<(), B::Error>;fn encode_add(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioAddArguments<'_, B>,) -> Result<(), B::Error>;fn encode_scale(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioScaleArguments<'_, B>,) -> Result<(), B::Error>;fn encode_clamp(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,arguments: AudioClampArguments<'_, B>,) -> Result<(), B::Error>;}diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsindex 839b637ea..78860d0e3 100644--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -70,7 +70,7 @@ impl<B: Backend> KVCacheUpdate<B> {in_place_data: &[KVLayerData<B>],source_indices: &[usize],destination_indices: &[usize],- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) -> Result<(), KVCacheUpdateError<B>> {if source_indices.len() != destination_indices.len() {return Err(KVCacheUpdateError::IndicesCountMismatch);diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rsindex 22476608f..8d5423b4d 100644--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs@@ -23,7 +23,7 @@ pub trait FullPrecisionMatmulKernel: Sized {fn encode(&mut self,context: &<Self::Backend as Backend>::Context,- encoder: &<Self::Backend as Backend>::ComputeEncoder,+ encoder: &mut <Self::Backend as Backend>::ComputeEncoder,arguments: FullPrecisionMatmulArguments<Self::Backend>,);}diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rsindex 0d4b30556..095ce436a 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs@@ -65,7 +65,7 @@ wherecontext: &B::Context,arguments: &MatmulArguments<B>,dispatch_descriptor: &DispatchDescriptor,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) -> Result<(), B::Error> {let config = dispatch_descriptor.specialization;diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rsindex 950fc9980..5bee72487 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/kernel.rs@@ -66,7 +66,7 @@ wherecontext: &B::Context,arguments: &MatmulArguments<B>,dispatch_descriptor: &DispatchDescriptor,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) -> Result<(), B::Error> {let config = dispatch_descriptor.specialization;let pipeline = self.get_or_create_kernel(context, config)?;diff --git a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/kernel.rsindex 2473e601a..a3f3588f0 100644--- a/crates/uzu/src/backends/common/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/kernel.rs@@ -75,7 +75,7 @@ wherecontext: &B::Context,arguments: &MatmulArguments<B>,dispatch_descriptor: &MatmulDispatchDescriptor,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) -> Result<(), B::Error> {match dispatch_descriptor {MatmulDispatchDescriptor::Gemv(d) => {@@ -98,7 +98,7 @@ wherecontext: &B::Context,arguments: MatmulArguments<B>,dispatch_descriptor: &MatmulDispatchDescriptor,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) -> Result<(), B::Error> {self.encode_dispatch_descriptor(context, &arguments, dispatch_descriptor, encoder)?;@@ -116,7 +116,7 @@ wherecontext: &B::Context,arguments: &MatmulArguments<B>,bias: &B::NativeBuffer,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) -> Result<(), B::Error> {let m = arguments.batch as usize;let n = arguments.output_dim as usize;diff --git a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rsindex 5506798bb..91f50eb46 100644--- a/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/split_k/kernel.rs@@ -63,7 +63,7 @@ wherecontext: &B::Context,arguments: &MatmulArguments<B>,dispatch_descriptor: &DispatchDescriptor,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) -> Result<(), B::Error> {self.ensure_kernels(context)?;self.ensure_accumulator_buffer(context, dispatch_descriptor.accumulator_bytes)?;diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rsindex 7139b277c..1f28e0aed 100644--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -27,7 +27,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {pub fn encode(&self,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,fused_up: &B::NativeBuffer,hidden: &B::NativeBuffer,m: i32,diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rsindex 70ca68396..e59cebd61 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs@@ -109,7 +109,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {args.gate_clip_max,args.up_clip_min,args.up_clip_max,- &encoder,+ encoder,);});@@ -126,7 +126,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {args.d_model as u32,args.d_ff as u32,args.k as u32,- &encoder,+ encoder,);});}diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex 44eeb1fba..305749f35 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -161,7 +161,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {pub fn encode(&self,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,arguments: QuantizedMatmulArguments<B>,) -> Result<(), QuantizedMatmulError<B>> {if arguments.quantization_type != self.quantization_type {diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rsindex eb62dc628..7f45c5196 100644--- a/crates/uzu/src/backends/common/kernel/sampling.rs+++ b/crates/uzu/src/backends/common/kernel/sampling.rs@@ -144,7 +144,7 @@ impl<B: Backend> SamplingKernel<B> {sampling_method: SamplingMethod,batch_size: usize,vocab_size: usize,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,) -> Result<(), SamplingError<B>> {if batch_size > self.max_batch_size {return Err(SamplingError::BatchSizeExceeded(batch_size, self.max_batch_size));diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rsindex b65adaa73..e478ba466 100644--- a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs@@ -55,7 +55,7 @@ impl<B: Backend> SSDPrefillKernels<B> {pub fn encode(&self,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,args: SSDPrefillArguments<B>,mode: SSDPrefillMode,) {diff --git a/crates/uzu/src/backends/cpu/backend.rs b/crates/uzu/src/backends/cpu/backend.rsindex ac7bd8408..6ac60bea7 100644--- a/crates/uzu/src/backends/cpu/backend.rs+++ b/crates/uzu/src/backends/cpu/backend.rs@@ -9,9 +9,9 @@ pub struct Cpu;impl Backend for Cpu {type Context = CpuContext;type NativeBuffer = Box<[u8]>;- type CommandBuffer = RefCell<CpuCommandBuffer>;- type ComputeEncoder = RefCell<CpuCommandBuffer>;- type CopyEncoder = RefCell<CpuCommandBuffer>;+ type CommandBuffer = CpuCommandBuffer;+ type ComputeEncoder = CpuCommandBuffer;+ type CopyEncoder = CpuCommandBuffer;type Event = RefCell<u64>;type Kernels = CpuKernels;type Error = CpuError;diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rsindex fa780d1e2..c627de861 100644--- a/crates/uzu/src/backends/cpu/command_buffer.rs+++ b/crates/uzu/src/backends/cpu/command_buffer.rs@@ -1,4 +1,7 @@-use std::{cell::RefCell, time::Instant};+use std::{+ cell::{OnceCell, RefCell},+ time::Instant,+};use super::Cpu;use crate::backends::common::{CommandBuffer, CopyEncoder};@@ -6,16 +9,16 @@ use crate::backends::common::{CommandBuffer, CopyEncoder};pub struct CpuCommandBuffer {commands: Vec<Box<dyn Fn()>>,completion_handlers: Vec<Box<dyn Fn()>>,- gpu_execution_time_ms: Option<f64>,+ gpu_execution_time_ms: OnceCell<f64>,}impl CpuCommandBuffer {- pub fn new() -> RefCell<CpuCommandBuffer> {- RefCell::new(CpuCommandBuffer {+ pub fn new() -> CpuCommandBuffer {+ CpuCommandBuffer {commands: Vec::new(),completion_handlers: Vec::new(),- gpu_execution_time_ms: None,- })+ gpu_execution_time_ms: OnceCell::new(),+ }}pub fn push_command(@@ -26,19 +29,19 @@ impl CpuCommandBuffer {}}-impl CommandBuffer for RefCell<CpuCommandBuffer> {+impl CommandBuffer for CpuCommandBuffer {type Backend = Cpu;fn with_compute_encoder<T>(&mut self,- callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,+ callback: impl FnOnce(&mut CpuCommandBuffer) -> T,) -> T {callback(self)}fn with_copy_encoder<T>(&mut self,- callback: impl FnOnce(&mut RefCell<CpuCommandBuffer>) -> T,+ callback: impl FnOnce(&mut CpuCommandBuffer) -> T,) -> T {callback(self)}@@ -49,7 +52,7 @@ impl CommandBuffer for RefCell<CpuCommandBuffer> {value: u64,) {let event = event.as_ptr();- self.borrow_mut().push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))+ self.push_command(Box::new(move || unsafe { assert!(*event >= value, "deadlock") }))}fn encode_signal_event(@@ -58,23 +61,25 @@ impl CommandBuffer for RefCell<CpuCommandBuffer> {value: u64,) {let event = event.as_ptr();- self.borrow_mut().push_command(Box::new(move || unsafe { *event = value }))+ self.push_command(Box::new(move || unsafe { *event = value }))}fn add_completion_handler(&mut self,handler: impl Fn() + 'static,) {- self.borrow_mut().completion_handlers.push(Box::new(move || handler()))+ self.completion_handlers.push(Box::new(move || handler()))}fn submit(&self) {let start = Instant::now();- for command in &self.borrow().commands {+ for command in &self.commands {command()}- self.borrow_mut().gpu_execution_time_ms = Some((Instant::now() - start).as_secs_f64() * 1000.0);- for completion_handler in self.borrow().completion_handlers.iter() {+ self.gpu_execution_time_ms+ .set((Instant::now() - start).as_secs_f64() * 1000.0)+ .expect("gpu execution time already set");+ for completion_handler in self.completion_handlers.iter() {completion_handler();}}@@ -82,11 +87,11 @@ impl CommandBuffer for RefCell<CpuCommandBuffer> {fn wait_until_completed(&self) {}fn gpu_execution_time_ms(&self) -> Option<f64> {- self.borrow().gpu_execution_time_ms+ self.gpu_execution_time_ms.get().copied()}}-impl CopyEncoder for RefCell<CpuCommandBuffer> {+impl CopyEncoder for CpuCommandBuffer {type Backend = Cpu;fn encode_copy(@@ -97,7 +102,7 @@ impl CopyEncoder for RefCell<CpuCommandBuffer> {) {let src = src.as_ptr();let dst = dst.as_ptr() as *mut u8;- self.borrow_mut().push_command(Box::new(move || unsafe {+ self.push_command(Box::new(move || unsafe {std::ptr::copy(src, dst, size);}))}@@ -110,7 +115,7 @@ impl CopyEncoder for RefCell<CpuCommandBuffer> {) {let size = range.end - range.start;let dst = dst[range].as_ptr() as *mut u8;- self.borrow_mut().push_command(Box::new(move || unsafe {+ self.push_command(Box::new(move || unsafe {dst.write_bytes(value, size);}))}diff --git a/crates/uzu/src/backends/cpu/context.rs b/crates/uzu/src/backends/cpu/context.rsindex cbf2d0aa2..4efd26ec3 100644--- a/crates/uzu/src/backends/cpu/context.rs+++ b/crates/uzu/src/backends/cpu/context.rs@@ -31,7 +31,7 @@ impl Context for CpuContext {Ok(vec![0; size].into_boxed_slice())}- fn create_command_buffer(&self) -> Result<RefCell<CpuCommandBuffer>, CpuError> {+ fn create_command_buffer(&self) -> Result<CpuCommandBuffer, CpuError> {Ok(CpuCommandBuffer::new())}diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsindex ec3bb5c04..406210962 100644--- a/crates/uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -27,14 +27,10 @@ include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));use crate::{DataType,backends::{- common::{- Backend,- kernel::matmul::{- FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,- MatmulKernels,- },+ common::kernel::matmul::{+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait, MatmulKernels,},- cpu::{Cpu, context::CpuContext, error::CpuError},+ cpu::{Cpu, command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError},},};@@ -53,10 +49,10 @@ impl FullPrecisionMatmulKernelTrait for FullPrecisionMatmulCpuKernel {fn encode(&mut self,#[allow(unused)] context: &CpuContext,- #[allow(unused)] encoder: &<Cpu as Backend>::ComputeEncoder,+ #[allow(unused)] encoder: &mut CpuCommandBuffer,#[allow(unused)] arguments: FullPrecisionMatmulArguments<Cpu>,) {- encoder.borrow_mut().push_command(move || todo!());+ encoder.push_command(move || todo!());}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex b13b6a219..33395af71 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -29,7 +29,7 @@ impl FullPrecisionMatmulKernelTrait for MatmulKernel<Metal> {fn encode(&mut self,context: &MetalContext,- encoder: &<Metal as crate::backends::common::Backend>::ComputeEncoder,+ encoder: &mut <Metal as crate::backends::common::Backend>::ComputeEncoder,arguments: FullPrecisionMatmulArguments<Metal>,) {let mut matmul_arguments = MatmulArguments {diff --git a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rsindex b231b3219..9c6e5cf95 100644--- a/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs+++ b/crates/uzu/src/backends/metal/metal_extensions/compute_command_encoder_extensions_conditional.rs@@ -20,17 +20,17 @@ trait ComputeEncoderRawConditional {/// Safe conditional wrapper that uses the raw unsafe methods internally.pub trait ComputeEncoderConditional {fn condition<IfBlock, ElseBlock>(- &self,+ &mut self,predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,offset: usize,if_block: IfBlock,else_block: Option<ElseBlock>,) where- IfBlock: FnOnce(),- ElseBlock: FnOnce();+ IfBlock: FnOnce(&mut Self),+ ElseBlock: FnOnce(&mut Self);}-impl ComputeEncoderRawConditional for ProtocolObject<dyn MTLComputeCommandEncoder> {+impl ComputeEncoderRawConditional for Retained<ProtocolObject<dyn MTLComputeCommandEncoder>> {fn encode_start_if(&self,predicate: &ProtocolObject<dyn MTLBuffer>,@@ -64,20 +64,20 @@ whereT: ComputeEncoderRawConditional,{fn condition<IfBlock, ElseBlock>(- &self,+ &mut self,predicate: &Retained<ProtocolObject<dyn MTLBuffer>>,offset: usize,if_block: IfBlock,else_block: Option<ElseBlock>,) where- IfBlock: FnOnce(),- ElseBlock: FnOnce(),+ IfBlock: FnOnce(&mut Self),+ ElseBlock: FnOnce(&mut Self),{self.encode_start_if(predicate, offset, MTLCompareFunction::Equal, 0);- if_block();+ if_block(self);if let Some(else_block) = else_block {self.encode_start_else();- else_block();+ else_block(self);}self.encode_end_if();}diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rsindex 133866eaa..59e0bc5bf 100644--- a/crates/uzu/src/encodable_block/activation.rs+++ b/crates/uzu/src/encodable_block/activation.rs@@ -47,7 +47,7 @@ impl<B: Backend> EncodableBlock<B> for Activation<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let input_array = arrays[0].borrow_mut();diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex 444127f32..5a6412e4a 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -134,7 +134,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {parameters: &EncodingParameters<B>,command_buffer: &mut B::CommandBuffer,) {- command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, &encoder));+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));if parameters.wait_until_completed {command_buffer.submit();@@ -150,7 +150,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- compute_encoder: &B::ComputeEncoder,+ compute_encoder: &mut B::ComputeEncoder,) {let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);@@ -260,7 +260,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {suffix_length as u32,0u32,max_sequence_length as u32,- &compute_encoder,+ compute_encoder,);}@@ -331,7 +331,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {suffix_length as u32,segment_prefix_length as u32,max_sequence_length as u32,- &compute_encoder,+ compute_encoder,);}@@ -401,7 +401,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {sinks_buffer,num_heads as u32,suffix_length as u32,- &compute_encoder,+ compute_encoder,)},KernelVariant::TwoPass => {@@ -434,7 +434,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {mask_q_seq_stride_opt,mask_head_stride_opt,sinks_buffer,- &compute_encoder,+ compute_encoder,);kernel_pass2.encode(partials_buf_borrow.deref(),@@ -443,7 +443,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {attention_output_buf_borrow.deref(),num_heads as u32,suffix_length as u32,- &compute_encoder,+ compute_encoder,);},}diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex e2389ee61..c1c1b8a8d 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -382,7 +382,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {debug_assert!(self.supports_shared_encoder(),diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex ac62d59fc..39adbb7eb 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -166,7 +166,7 @@ impl<B: Backend> EncodableBlock<B> for Decoder<B> {&self,_state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- _encoder: &B::ComputeEncoder,+ _encoder: &mut B::ComputeEncoder,) {unimplemented!("Decoder does not support shared encoder")}diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rsindex 910273e7e..95d9a79d8 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs@@ -95,7 +95,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rsindex 5342b58d3..d55aada5d 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs@@ -99,7 +99,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);let batch_size = state.sampling_length();diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rsindex 6b748773a..237b7f402 100644--- a/crates/uzu/src/encodable_block/full_precision_linear.rs+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs@@ -131,7 +131,7 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex f49b3dc76..af8d857e4 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -429,7 +429,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {debug_assert!(self.supports_shared_encoder(),diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rsindex 3992419bb..47376d891 100644--- a/crates/uzu/src/encodable_block/layer_norm.rs+++ b/crates/uzu/src/encodable_block/layer_norm.rs@@ -74,7 +74,7 @@ impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex f9e0c1fc2..1746a3152 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -138,7 +138,7 @@ impl<B: Backend> MambaMixer<B> {fn encode_pipeline_with_encoder(&self,state: &mut ForwardPassState<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,parameters: &EncodingParameters<B>,) {let active_suffix_length = state.active_suffix_length();@@ -162,7 +162,7 @@ impl<B: Backend> MambaMixer<B> {fn run_split_inproj(&self,state: &mut ForwardPassState<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,suffix_length: usize,) {let arrays = state.arrays(&[@@ -202,7 +202,7 @@ impl<B: Backend> MambaMixer<B> {fn run_conv_scan(&self,state: &mut ForwardPassState<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,suffix_length: usize,) {let arrays = state.arrays(&[@@ -246,7 +246,7 @@ impl<B: Backend> MambaMixer<B> {suffix_length as u32,inner_dim as u32,proj_dim as u32,- &encoder,+ encoder,);}} else {@@ -262,7 +262,7 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,suffix_length as u32,conv_dim as u32,- &encoder,+ encoder,);Some(buffer)@@ -290,7 +290,7 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,inner_dim as u32,proj_dim as u32,- &encoder,+ encoder,)}}@@ -299,7 +299,7 @@ impl<B: Backend> MambaMixer<B> {fn run_prefill_ssm(&self,state: &mut ForwardPassState<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,suffix_length: usize,) {let base_arrays = state.arrays(&[@@ -350,7 +350,7 @@ impl<B: Backend> MambaMixer<B> {fn run_decode_ssm(&self,state: &mut ForwardPassState<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,suffix_length: usize,) {let arrays = state.arrays(&[@@ -481,7 +481,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {self.encode_pipeline_with_encoder(state, encoder, parameters);}diff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rsindex 17bf482de..3da5af103 100644--- a/crates/uzu/src/encodable_block/mlp.rs+++ b/crates/uzu/src/encodable_block/mlp.rs@@ -75,7 +75,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {&self,state: &mut ForwardPassState<B>,params: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {// Upself.up.encode_with_shared_encoder(state, params, encoder);diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex d1ae53711..2f1abc358 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -107,7 +107,7 @@ pub trait EncodableBlock<B: Backend> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,);fn supports_shared_encoder(&self) -> bool {diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rsindex 61a4636ae..56bd2b891 100644--- a/crates/uzu/src/encodable_block/moe_block.rs+++ b/crates/uzu/src/encodable_block/moe_block.rs@@ -308,7 +308,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {e as u32,k as u32,self.router_renorm,- &encoder,+ encoder,);});}@@ -322,7 +322,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {suffix_length as u32,e as u32,k as u32,- &encoder,+ encoder,);});@@ -337,7 +337,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {num_blocks as u32,num_tiles as u32,0u32,- &encoder,+ encoder,);});command_buffer.with_compute_encoder(|encoder| {@@ -355,7 +355,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {num_blocks as u32,num_tiles as u32,tok2row_buf.borrow().deref(),- &encoder,+ encoder,);});@@ -476,7 +476,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {suffix_length as u32,self.model_dim as u32,k as u32,- &encoder,+ encoder,);});@@ -494,7 +494,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {&self,_state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- _encoder: &B::ComputeEncoder,+ _encoder: &mut B::ComputeEncoder,) {unreachable!("MoeBlock does not support shared compute encoder");}diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rsindex b040f39af..0553ecef4 100644--- a/crates/uzu/src/encodable_block/normalization.rs+++ b/crates/uzu/src/encodable_block/normalization.rs@@ -70,7 +70,7 @@ impl<B: Backend> EncodableBlock<B> for Normalization<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {match self {Self::LayerNorm(layer_norm) => layer_norm.encode_with_shared_encoder(state, parameters, encoder),diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rsindex 1424c6013..f5c75e496 100644--- a/crates/uzu/src/encodable_block/pooling.rs+++ b/crates/uzu/src/encodable_block/pooling.rs@@ -26,7 +26,7 @@ impl<B: Backend> PoolingKernel<B> {seq_len: u32,hidden_dim: u32,batch_size: u32,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {match self {Self::Cls(kernel) => kernel.encode(input, output, seq_len, hidden_dim, batch_size, encoder),@@ -84,7 +84,7 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let batch_size = 1;let seq_len = state.aux_buffers_suffix_length();diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rsindex 6cbca8760..0b5bf96aa 100644--- a/crates/uzu/src/encodable_block/prediction_head.rs+++ b/crates/uzu/src/encodable_block/prediction_head.rs@@ -78,7 +78,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {self.dense.encode_with_shared_encoder(state, parameters, encoder);self.activation.encode_with_shared_encoder(state, parameters, encoder);diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rsindex 4d51f0932..cbadad192 100644--- a/crates/uzu/src/encodable_block/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -126,7 +126,7 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let qkv_binding = state.arrays(&[self.qkv_array_id]);let qkv_shape = {diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rsindex 7d27262f4..f9675456c 100644--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs+++ b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs@@ -243,7 +243,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rsindex 1576d8d8c..b63dd7305 100644--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs@@ -233,7 +233,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);let batch_size = state.sampling_length();diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rsindex fa8f4fdd4..da89dd758 100644--- a/crates/uzu/src/encodable_block/quantized_linear.rs+++ b/crates/uzu/src/encodable_block/quantized_linear.rs@@ -248,7 +248,7 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex a1b7cea4c..433646f55 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -88,7 +88,7 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let input_binding = state.arrays(&[self.input_array_id]);let output_binding = state.arrays(&[self.output_array_id]);diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rsindex 2cbd65be8..dc450d574 100644--- a/crates/uzu/src/encodable_block/rope.rs+++ b/crates/uzu/src/encodable_block/rope.rs@@ -39,7 +39,7 @@ impl<B: Backend> EncodableBlock<B> for Rope<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex e3644d572..846cf8b22 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -49,7 +49,7 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex dccc21457..e4a715d45 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -116,7 +116,7 @@ impl<B: Backend> ShortConvMixer<B> {self.in_projection.encode(state, parameters, command_buffer);- command_buffer.with_compute_encoder(|encoder| self.run_conv(state, &encoder, active_suffix_length));+ command_buffer.with_compute_encoder(|encoder| self.run_conv(state, encoder, active_suffix_length));self.out_projection.encode(state, parameters, command_buffer);@@ -129,7 +129,7 @@ impl<B: Backend> ShortConvMixer<B> {fn encode_pipeline_with_encoder(&self,state: &mut ForwardPassState<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,parameters: &EncodingParameters<B>,) {let active_suffix_length = state.active_suffix_length();@@ -173,7 +173,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_conv(&self,state: &mut ForwardPassState<B>,- compute: &B::ComputeEncoder,+ compute: &mut B::ComputeEncoder,active_suffix_length: usize,) {self.clear_suffix_state_valid_range(state);@@ -212,7 +212,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_prefill_conv(&self,state: &mut ForwardPassState<B>,- compute: &B::ComputeEncoder,+ compute: &mut B::ComputeEncoder,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -269,7 +269,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_trie_conv(&self,state: &mut ForwardPassState<B>,- compute: &B::ComputeEncoder,+ compute: &mut B::ComputeEncoder,sampling_start: usize,trie_len: usize,) {@@ -328,7 +328,7 @@ impl<B: Backend> ShortConvMixer<B> {fn run_decode_conv(&self,state: &mut ForwardPassState<B>,- compute: &B::ComputeEncoder,+ compute: &mut B::ComputeEncoder,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -374,7 +374,7 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {) {if self.supports_shared_encoder() {command_buffer- .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, &encoder, parameters));+ .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));if parameters.wait_until_completed {command_buffer.submit();@@ -393,7 +393,7 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {self.encode_pipeline_with_encoder(state, encoder, parameters);}diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rsindex 9aa6ba3da..284794698 100644--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs@@ -40,7 +40,7 @@ impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rsindex 211c398a1..1c9312274 100644--- a/crates/uzu/src/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/encodable_block/tensor_copy.rs@@ -40,7 +40,7 @@ impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {&self,state: &mut ForwardPassState<B>,_parameters: &EncodingParameters<B>,- encoder: &B::ComputeEncoder,+ encoder: &mut B::ComputeEncoder,) {let arrays = state.arrays(&self.argument_arrays);assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex dd27362c8..fad69fc8f 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -553,7 +553,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {if update.unmask_col >= 0 || update.mask_col >= 0 {let mask_buf_rc = mask_buffer.borrow().buffer();let mask_buf_borrow = mask_buf_rc.borrow();- mask_update.encode(mask_buf_borrow.deref(), update.unmask_col, update.mask_col, &encoder);+ mask_update.encode(mask_buf_borrow.deref(), update.unmask_col, update.mask_col, encoder);}}}diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex 085e0546c..8aa469b44 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -317,7 +317,7 @@ fn run_single_pass_attention(.expect("Failed to create buffer");let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -347,7 +347,7 @@ fn run_single_pass_attention(sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();@@ -390,7 +390,7 @@ fn run_single_pass_attention_with_is_causal(.expect("Failed to create buffer");let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -419,7 +419,7 @@ fn run_single_pass_attention_with_is_causal(sinks_buffer.as_ref().map(|b| b),num_heads as u32,seq_len as u32,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();@@ -461,7 +461,7 @@ fn run_gemm_attention(.expect("Failed to create buffer");let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let args = AttentionGemmArguments {queries_buffer: &query_buffer,@@ -481,7 +481,7 @@ fn run_gemm_attention(scale,};- let encode_result = kernel.encode(context, &compute_encoder, &args);+ let encode_result = kernel.encode(context, &mut compute_encoder, &args);compute_encoder.end_encoding();encode_result?;@@ -936,7 +936,7 @@ fn run_two_pass_attention(.expect("Failed to create buffer");let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let mut mask_kv_seq_stride: Option<u32> = None;let mut mask_q_seq_stride: Option<u32> = None;@@ -967,7 +967,7 @@ fn run_two_pass_attention(mask_q_seq_stride,mask_head_stride,sinks_buffer.as_ref().map(|b| b),- &compute_encoder,+ &mut compute_encoder,);kernel_pass2.encode(&partials_buffer,@@ -976,7 +976,7 @@ fn run_two_pass_attention(&output_buffer,num_heads as u32,seq_len as u32,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();@@ -1174,7 +1174,7 @@ fn perf_two_pass_attention() {// ---- Launch and time ----let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer");- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");let mask_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;let sinks_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>> = None;@@ -1199,7 +1199,7 @@ fn perf_two_pass_attention() {None,None,sinks_buffer.as_ref().map(|b| b),- &compute_encoder,+ &mut compute_encoder,);kernel_pass2.encode(&partials_buffer,@@ -1208,7 +1208,7 @@ fn perf_two_pass_attention() {&output_buffer,num_heads as u32,suffix_length as u32,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rsindex 3075e4dbb..8e5348c2d 100644--- a/crates/uzu/tests/audio_codec_test.rs+++ b/crates/uzu/tests/audio_codec_test.rs@@ -99,7 +99,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {}let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");kernel.encode(input.buffer(),@@ -117,7 +117,7 @@ fn audio_conv1d_replicate_matches_reference_f32() {padding as i32,1_i32,batch_size as i32,- &encoder,+ &mut encoder,);encoder.end_encoding();@@ -205,7 +205,7 @@ fn audio_causal_conv1d_matches_reference_f32() {}let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");kernel.encode(input.buffer(),@@ -219,7 +219,7 @@ fn audio_causal_conv1d_matches_reference_f32() {kernel_size as i32,dilation as i32,batch_size as i32,- &encoder,+ &mut encoder,);encoder.end_encoding();@@ -320,7 +320,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {}let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");kernel.encode(input.buffer(),@@ -335,7 +335,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {stride as i32,groups as i32,batch_size as i32,- &encoder,+ &mut encoder,);encoder.end_encoding();@@ -370,9 +370,9 @@ fn audio_leaky_relu_matches_reference_f32() {let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");- kernel.encode(input.buffer(), output.buffer(), n as i32, slope, &encoder);+ kernel.encode(input.buffer(), output.buffer(), n as i32, slope, &mut encoder);encoder.end_encoding();command_buffer.commit();@@ -405,9 +405,9 @@ fn audio_tanh_matches_reference_f32() {let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");- kernel.encode(input.buffer(), output.buffer(), n as i32, &encoder);+ kernel.encode(input.buffer(), output.buffer(), n as i32, &mut encoder);encoder.end_encoding();command_buffer.commit();@@ -444,11 +444,11 @@ fn audio_add_and_scale_match_reference_f32() {let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");- add_kernel.encode(a.buffer(), b.buffer(), sum.buffer(), n as i32, &encoder);+ add_kernel.encode(a.buffer(), b.buffer(), sum.buffer(), n as i32, &mut encoder);- scale_kernel.encode(sum.buffer(), scaled.buffer(), n as i32, scale_value, &encoder);+ scale_kernel.encode(sum.buffer(), scaled.buffer(), n as i32, scale_value, &mut encoder);encoder.end_encoding();command_buffer.commit();@@ -494,9 +494,9 @@ fn audio_clamp_matches_reference_f32() {let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");- kernel.encode(input.buffer(), output.buffer(), n as i32, min_value, max_value, &encoder);+ kernel.encode(input.buffer(), output.buffer(), n as i32, min_value, max_value, &mut encoder);encoder.end_encoding();command_buffer.commit();@@ -537,7 +537,7 @@ fn audio_half_snake_matches_reference_f32() {let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");kernel.encode(input.buffer(),@@ -549,7 +549,7 @@ fn audio_half_snake_matches_reference_f32() {negative_slope,eps,batch_size as i32,- &encoder,+ &mut encoder,);encoder.end_encoding();@@ -608,7 +608,7 @@ fn audio_fsq_decode_matches_reference() {context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");kernel.encode(tokens.buffer(),@@ -619,7 +619,7 @@ fn audio_fsq_decode_matches_reference() {codebook_dim as i32,&num_levels,batch_size as i32,- &encoder,+ &mut encoder,);encoder.end_encoding();@@ -671,7 +671,7 @@ fn audio_fsq_encode_matches_reference() {let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");let command_buffer = context.command_queue.command_buffer().expect("command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");kernel.encode(input.buffer(),@@ -684,7 +684,7 @@ fn audio_fsq_encode_matches_reference() {&dim_base_index,eps,batch_size as i32,- &encoder,+ &mut encoder,);encoder.end_encoding();diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 4bf11fe20..bb2e70e9c 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -46,7 +46,7 @@ fn run_metal_matmul(let mut kernel = MatmulKernel::<Metal>::new(DataType::BF16).expect("kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");let mut arguments = MatmulArguments {a: &a_buf,a_offset: 0,@@ -64,7 +64,7 @@ fn run_metal_matmul(};MatmulKernel::<Metal>::apply_batch_collapse(&mut arguments);let descriptor = choose_dispatch_descriptor(ctx, DataType::BF16, &arguments).expect("dispatch descriptor");- let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &enc);+ let encode_result = kernel.encode_with_descriptor(ctx, arguments, &descriptor, &mut enc);enc.end_encoding();encode_result.expect("encode");cb.commit();diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rsindex 85cb7bb02..78e3aab7e 100644--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs@@ -76,7 +76,7 @@ fn gen_topk_ids_from_logits(// Use fused router+topk kernellet cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");+ let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router_topk encoder");let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(ctx, DataType::BF16).expect("router_topk kernel");router_topk.encode(@@ -90,7 +90,7 @@ fn gen_topk_ids_from_logits(e as u32,k as u32,true,- &router_topk_encoder,+ &mut router_topk_encoder,);router_topk_encoder.end_encoding();cb.commit();@@ -127,7 +127,7 @@ fn test_counts_offsets_fused_parity_random() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");kernel.encode(&topk_ids_buf,@@ -137,7 +137,7 @@ fn test_counts_offsets_fused_parity_random() {t as u32,e as u32,k as u32,- &encoder,+ &mut encoder,);encoder.end_encoding();@@ -179,8 +179,8 @@ fn test_counts_offsets_fused_edge_cases() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");- kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &mut encoder);encoder.end_encoding();cb.commit();cb.wait_until_completed();@@ -203,8 +203,8 @@ fn test_counts_offsets_fused_edge_cases() {let kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");- kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ kernel.encode(&topk_ids_buf, &offsets_buf, &sum_k_buf, &partials_buf, t as u32, e as u32, k as u32, &mut encoder);encoder.end_encoding();cb.commit();cb.wait_until_completed();diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex 874a4b28b..fb5fdf845 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -351,7 +351,7 @@ fn run_moe_parity_test_internal(// Router + TopK (fused kernel)let router_topk =<<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router+topk");- let router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");+ let mut router_topk_encoder = cb.new_compute_command_encoder().expect("router+topk_encoder");router_topk.encode(&x_buf,&router_w_buf,@@ -363,17 +363,26 @@ fn run_moe_parity_test_internal(e as u32,k as u32,true,- &router_topk_encoder,+ &mut router_topk_encoder,);router_topk_encoder.end_encoding();let fused_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");- let encoder = cb.new_compute_command_encoder().expect("encoder");- fused_kernel.encode(&topk_ids_buf, &offsets_buf, &sumk_buf, &partials_buf, t as u32, e as u32, k as u32, &encoder);+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");+ fused_kernel.encode(+ &topk_ids_buf,+ &offsets_buf,+ &sumk_buf,+ &partials_buf,+ t as u32,+ e as u32,+ k as u32,+ &mut encoder,+ );encoder.end_encoding();- let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");+ let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");let scatter_bases_kernel = <<Metal as Backend>::Kernels as Kernels>::MoeBlockBasesFromPartialsKernel::new(&ctx).expect("scatter bases kernel");scatter_bases_kernel.encode(@@ -384,11 +393,11 @@ fn run_moe_parity_test_internal(num_blocks as u32,num_tiles as u32,0u32,- &scatter_bases_encoder,+ &mut scatter_bases_encoder,);scatter_bases_encoder.end_encoding();- let scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");+ let mut scatter_encoder = cb.new_compute_command_encoder().expect("scatter encoder");let scatter_map_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel::new(&ctx, DataType::BF16).expect("Failed to create <<Metal as Backend>::Kernels as Kernels>::MoeScatterBucketsMapKernel");@@ -406,7 +415,7 @@ fn run_moe_parity_test_internal(num_blocks as u32,num_tiles as u32,&tok2row_buf,- &scatter_encoder,+ &mut scatter_encoder,);scatter_encoder.end_encoding();@@ -464,7 +473,7 @@ fn run_moe_parity_test_internal(let finalize =<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");finalize.encode(&tok2row_buf,&topk_probs_buf,@@ -473,7 +482,7 @@ fn run_moe_parity_test_internal(t as u32,d_model as u32,k as u32,- &encoder,+ &mut encoder,);encoder.end_encoding();diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rsindex 77254d876..077e7c797 100644--- a/crates/uzu/tests/moe/moe_finalize_test.rs+++ b/crates/uzu/tests/moe/moe_finalize_test.rs@@ -91,7 +91,7 @@ fn test_finalize_correctness() {let finalize = <<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");finalize.encode(&tok2row_buf,&probs_buf,@@ -100,7 +100,7 @@ fn test_finalize_correctness() {t as u32,d_model as u32,k as u32,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex f62468bb1..fed88bc9a 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -122,7 +122,7 @@ fn test_moe_e2e_decode_perf() {// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");router_topk.encode(&x_buf,&router_w_buf,@@ -134,7 +134,7 @@ fn test_moe_e2e_decode_perf() {e as u32,k as u32,true,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();@@ -183,7 +183,7 @@ fn test_moe_e2e_prefill_perf() {// Time fused Router+TopKlet fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");router_topk.encode(&x_buf,&router_w_buf,@@ -195,7 +195,7 @@ fn test_moe_e2e_prefill_perf() {e as u32,k as u32,true,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();@@ -312,7 +312,7 @@ fn test_moe_pipeline_breakdown_decode() {// Testing: Router + TopK + Counts+Offsets (FUSED)let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");router_topk_fused_kernel.encode(&x_buf,&router_w_buf,@@ -324,7 +324,7 @@ fn test_moe_pipeline_breakdown_decode() {e as u32,k as u32,true,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();@@ -333,7 +333,7 @@ fn test_moe_pipeline_breakdown_decode() {let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");counts_offsets_kernel.encode(&topk_ids_buf,&offsets_buf,@@ -342,7 +342,7 @@ fn test_moe_pipeline_breakdown_decode() {t as u32,e as u32,k as u32,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();@@ -351,7 +351,7 @@ fn test_moe_pipeline_breakdown_decode() {let scatter_perf = time_kernel("Scatter", 2, 5, || {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");+ let mut scatter_bases_encoder = cb.new_compute_command_encoder().expect("scatter_bases_encoder");scatter_bases_kernel.encode(&partials_buf,&block_bases_buf,@@ -360,11 +360,11 @@ fn test_moe_pipeline_breakdown_decode() {num_blocks as u32,num_tiles as u32,0u32,- &scatter_bases_encoder,+ &mut scatter_bases_encoder,);scatter_bases_encoder.end_encoding();- let scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");+ let mut scatter_map_encoder = cb.new_compute_command_encoder().expect("scatter_map_encoder");scatter_map_kernel.encode(&topk_ids_buf,&topk_probs_buf,@@ -379,7 +379,7 @@ fn test_moe_pipeline_breakdown_decode() {num_blocks as u32,num_tiles as u32,&tok2row_buf,- &scatter_map_encoder,+ &mut scatter_map_encoder,);scatter_map_encoder.end_encoding();@@ -445,7 +445,7 @@ fn test_moe_pipeline_breakdown_decode() {let finalize_perf = time_kernel("Finalize", 2, 5, || {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");finalize_kernel.encode(&tok2row_buf,&topk_probs_buf,@@ -454,7 +454,7 @@ fn test_moe_pipeline_breakdown_decode() {t as u32,d_model as u32,k as u32,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rsindex d5145c00c..9932b651e 100644--- a/crates/uzu/tests/moe/moe_router_topk_test.rs+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs@@ -158,7 +158,7 @@ fn run_router_topk_once(let probs_buf = alloc_buffer::<bf16>(ctx, t * k);let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("Failed to create command encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("Failed to create command encoder");kernel.encode(&input_buf,&weight_buf,@@ -170,7 +170,7 @@ fn run_router_topk_once(e as u32,k as u32,renorm,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rsindex f47893c00..a369da7af 100644--- a/crates/uzu/tests/moe/moe_scatter_test.rs+++ b/crates/uzu/tests/moe/moe_scatter_test.rs@@ -85,7 +85,7 @@ fn test_scatter_buckets_parity() {let router_topk = <<Metal as Backend>::Kernels as Kernels>::MoeRouterTopKKernel::new(&ctx, DataType::BF16).expect("router_topk");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("Failed to create encoder");router_topk.encode(&input_buf,&weight_buf,@@ -97,7 +97,7 @@ fn test_scatter_buckets_parity() {e as u32,k as u32,true,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();@@ -117,7 +117,7 @@ fn test_scatter_buckets_parity() {let fused_kernel =<<Metal as Backend>::Kernels as Kernels>::MoeCountsOffsetsFusedKernel::new(&ctx).expect("fused kernel");let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb.new_compute_command_encoder().expect("encoder");+ let mut encoder = cb.new_compute_command_encoder().expect("encoder");fused_kernel.encode(&topk_ids_buf,&offsets_buf,@@ -126,7 +126,7 @@ fn test_scatter_buckets_parity() {t as u32,e as u32,k as u32,- &encoder,+ &mut encoder,);encoder.end_encoding();cb.commit();@@ -146,7 +146,8 @@ fn test_scatter_buckets_parity() {let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let block_alloc_buf = alloc_buffer::<u32>(&ctx, entries);- let scatter_bases_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");+ let mut scatter_bases_encoder =+ cb.new_compute_command_encoder().expect("Failed to create scatter_bases_encoder");scatter_bases_kernel.encode(&partials_buf,&block_bases_buf,@@ -155,7 +156,7 @@ fn test_scatter_buckets_parity() {num_blocks as u32,num_tiles as u32,0u32,- &scatter_bases_encoder,+ &mut scatter_bases_encoder,);scatter_bases_encoder.end_encoding();@@ -167,7 +168,7 @@ fn test_scatter_buckets_parity() {let out_probs_buf = alloc_buffer::<bf16>(&ctx, sumk);let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");+ let mut scatter_encoder = cb.new_compute_command_encoder().expect("Failed to create scatter_encoder");scatter_kernel.encode(&topk_ids_buf,&topk_probs_buf,@@ -181,7 +182,7 @@ fn test_scatter_buckets_parity() {k as u32,num_blocks as u32,num_tiles as u32,- &scatter_encoder,+ &mut scatter_encoder,);scatter_encoder.end_encoding();diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsindex 17d4e7d40..69c25999c 100644--- a/crates/uzu/tests/quant_matmul_test.rs+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -511,8 +511,8 @@ fn execute_quantized_matmul(quantization_type,};let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&encoder, args).unwrap();+ let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&mut encoder, args).unwrap();encoder.end_encoding();cb_ref.commit();cb_ref.wait_until_completed();@@ -534,8 +534,8 @@ fn execute_quantized_matmul(quantization_type,};let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&encoder, args).unwrap();+ let mut encoder = cb_ref.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&mut encoder, args).unwrap();encoder.end_encoding();cb_ref.commit();cb_ref.wait_until_completed();diff --git a/crates/uzu/tests/rms_norm_kernel_test.rs b/crates/uzu/tests/rms_norm_kernel_test.rsindex 4f9b430f9..f44a409a8 100644--- a/crates/uzu/tests/rms_norm_kernel_test.rs+++ b/crates/uzu/tests/rms_norm_kernel_test.rs@@ -311,7 +311,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(// Create command buffer and encodelet command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&input_buffer,@@ -322,7 +322,7 @@ fn test_rms_norm_basic_typed<InputT, ScaleT, OutputT>(epsilon,0.0,false,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();@@ -428,7 +428,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&input_buffer,@@ -439,7 +439,7 @@ fn test_rms_norm_edge_cases_typed<InputT, ScaleT, OutputT>(epsilon,0.0,false,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();@@ -635,7 +635,7 @@ fn perf_rms_norm_with_size(// ---- Launch and time ----let command_buffer_ref = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&input_buffer,@@ -646,7 +646,7 @@ fn perf_rms_norm_with_size(EPSILON,0.0,false,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();@@ -802,7 +802,8 @@ fn qk_norm_test() {// Test Q head normalization{let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");q_kernel.encode(&qkv_buffer,@@ -817,7 +818,7 @@ fn qk_norm_test() {0,num_q_heads as u32,false,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();@@ -864,7 +865,8 @@ fn qk_norm_test() {// Test K head normalization{let command_buffer = mtl_context.command_queue.command_buffer().expect("Failed to create command buffer");- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");k_kernel.encode(&qkv_buffer,@@ -879,7 +881,7 @@ fn qk_norm_test() {num_q_heads as u32,num_kv_heads as u32,false,- &compute_encoder,+ &mut compute_encoder,);compute_encoder.end_encoding();diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex 231fd027f..6744ffcc6 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -111,7 +111,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {let command_buffer = command_buffer_ref.to_owned();// Run sampling- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(&logits_buffer,@@ -123,7 +123,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {SamplingMethod::Greedy,batch_size,vocab_size,- &compute_encoder,+ &mut compute_encoder,).expect("Argmax sampling should succeed");compute_encoder.end_encoding();@@ -240,7 +240,7 @@ fn test_topp_sampling_from_prob_exact_match(let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(&logits_buf,@@ -257,7 +257,7 @@ fn test_topp_sampling_from_prob_exact_match(},batch_size,vocab_size,- &compute_encoder,+ &mut compute_encoder,).expect("encode");compute_encoder.end_encoding();@@ -363,7 +363,7 @@ fn test_topp_sampling_statistical_large() {let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -381,7 +381,7 @@ fn test_topp_sampling_statistical_large() {},BATCH,VOCAB,- &compute_encoder,+ &mut compute_encoder,).expect("encode");compute_encoder.end_encoding();@@ -465,7 +465,7 @@ fn perf_topp_128k_vocab() {// ---- Launch once and time ----let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -483,7 +483,7 @@ fn perf_topp_128k_vocab() {},BATCH,VOCAB,- &compute_encoder,+ &mut compute_encoder,).expect("encode");compute_encoder.end_encoding();@@ -569,7 +569,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {// ---- Launch once and time ----let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -582,7 +582,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {SamplingMethod::Greedy,BATCH,VOCAB,- &compute_encoder,+ &mut compute_encoder,).expect("encode");compute_encoder.end_encoding();@@ -688,7 +688,8 @@ fn test_categorical_sampling() {let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -706,7 +707,7 @@ fn test_categorical_sampling() {},batch_size,vocab_size,- &compute_encoder,+ &mut compute_encoder,).expect("Categorical sampling should succeed");compute_encoder.end_encoding();@@ -829,7 +830,8 @@ fn test_categorical_sampling_statistical() {let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder =+ command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -847,7 +849,7 @@ fn test_categorical_sampling_statistical() {},BATCH,VOCAB,- &compute_encoder,+ &mut compute_encoder,).expect("encode");compute_encoder.end_encoding();@@ -935,7 +937,7 @@ fn perf_categorical_128k_vocab() {// Launch and timelet cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(@@ -953,7 +955,7 @@ fn perf_categorical_128k_vocab() {},BATCH,VOCAB,- &compute_encoder,+ &mut compute_encoder,).expect("encode");compute_encoder.end_encoding();@@ -1024,8 +1026,8 @@ fn test_temperature_gpu_cpu_match() {let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &compute_encoder);+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TEMPERATURE, &mut compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();@@ -1090,8 +1092,8 @@ fn test_topk_gpu_cpu_match() {let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &compute_encoder);+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPK, &mut compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();@@ -1155,8 +1157,8 @@ fn test_topp_gpu_cpu_match() {let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &compute_encoder);+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, TOPP, &mut compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();@@ -1225,8 +1227,8 @@ fn test_minp_gpu_cpu_match() {let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, MINP, &compute_encoder);+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&logits_buffer, &processed_buffer, BATCH as u32, VOCAB as u32, MINP, &mut compute_encoder);compute_encoder.end_encoding();command_buffer_ref.commit();@@ -1313,7 +1315,7 @@ fn test_minp_sampling_exact_match(let cb_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut compute_encoder = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode_with_encoder(&logits_buf,@@ -1330,7 +1332,7 @@ fn test_minp_sampling_exact_match(},batch_size,vocab_size,- &compute_encoder,+ &mut compute_encoder,).expect("encode");compute_encoder.end_encoding();@@ -1412,8 +1414,15 @@ fn test_gumbel_gpu_cpu_match() {let command_buffer_ref = context.command_queue.command_buffer().expect("Failed to create command buffer");let command_buffer = command_buffer_ref.to_owned();- let compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&logits_buffer, &seeds_buffer, &gumbel_logits_buffer, BATCH as u32, VOCAB as u32, &compute_encoder);+ let mut compute_encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(+ &logits_buffer,+ &seeds_buffer,+ &gumbel_logits_buffer,+ BATCH as u32,+ VOCAB as u32,+ &mut compute_encoder,+ );compute_encoder.end_encoding();command_buffer_ref.commit();diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex e33fbd8da..c0e4ffb54 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -249,8 +249,8 @@ fn run_prefill_kernel_mode(};let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- kernel.encode(&encoder, args, mode);+ let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ kernel.encode(&mut encoder, args, mode);encoder.end_encoding();command_buffer.commit();command_buffer.wait_until_completed();@@ -321,7 +321,7 @@ fn run_conv_scan_once(}let command_buffer = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&padded_buf,&w_buf,@@ -337,7 +337,7 @@ fn run_conv_scan_once(channels as u32,channels as u32,0u32,- &encoder,+ &mut encoder,);encoder.end_encoding();diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rsindex e952f8eb5..0552a5348 100644--- a/crates/uzu/tests/ssm_update_test.rs+++ b/crates/uzu/tests/ssm_update_test.rs@@ -241,7 +241,7 @@ fn ssd_update_with_z_bf16() {let kernel = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(&ctx, DataType::BF16).unwrap();let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");let cb = cb_ref.to_owned();- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");+ let mut enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");kernel.encode(&x_buf,&dt_buf,@@ -261,7 +261,7 @@ fn ssd_update_with_z_bf16() {bsz as u32,h as u32,dh as u32,- &enc,+ &mut enc,);enc.end_encoding();cb_ref.commit();