A high-performance inference engine for AI models
Backend structs mutability (#228)
Aleksandr Golokozc00ed108236d
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex b7f677d03..76060401e 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -320,13 +320,13 @@ pub fn bindgen(Ok(Self { pipeline #(, #conditional_buffer_sets)* })}- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder ProtocolObject<dyn MTLComputeCommandEncoder>) {+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {#empty_dispatch_guardscompute_encoder.set_compute_pipeline_state(&self.pipeline);#(#encode_args_sets)*#dispatch}- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder ProtocolObject<dyn MTLComputeCommandEncoder>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {+ fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {#empty_dispatch_guardsif let Some(predicate) = predicate {diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rsindex 730b54686..9b9b658d8 100644--- a/crates/uzu/src/array.rs+++ b/crates/uzu/src/array.rs@@ -9,7 +9,7 @@ use crate::{#[derive(Debug)]pub struct Array<B: Backend> {- buffer: Rc<B::NativeBuffer>,+ buffer: Rc<RefCell<B::NativeBuffer>>,offset: usize,shape: Box<[usize]>,data_type: DataType,@@ -18,20 +18,20 @@ pub struct Array<B: Backend> {impl<B: Backend> Array<B> {// Constructorspub unsafe fn from_parts(- buffer: Rc<B::NativeBuffer>,+ buffer: Rc<RefCell<B::NativeBuffer>>,offset: usize,shape: &[usize],data_type: DataType,) -> Self {let required_bytes = size_for_shape(shape, data_type);assert!(- offset + required_bytes <= buffer.length(),+ offset + required_bytes <= buffer.borrow().length(),"Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",shape,data_type,offset,offset + required_bytes,- buffer.length()+ buffer.borrow().length());Self {buffer: buffer.clone(),@@ -49,11 +49,7 @@ impl<B: Backend> Array<B> {}// Getters- pub fn buffer(&self) -> &B::NativeBuffer {- &self.buffer.as_ref()- }-- pub fn buffer_rc(&self) -> Rc<B::NativeBuffer> {+ pub fn buffer(&self) -> Rc<RefCell<B::NativeBuffer>> {self.buffer.clone()}@@ -71,7 +67,7 @@ impl<B: Backend> Array<B> {// Utilitypub fn cpu_ptr(&self) -> NonNull<c_void> {- unsafe { self.buffer.cpu_ptr().add(self.offset) }+ unsafe { self.buffer.borrow().cpu_ptr().add(self.offset) }}pub fn size(&self) -> usize {@@ -278,6 +274,6 @@ impl<C: Context> ArrayContextExt for C {let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");buffer.set_label(Some(label));- unsafe { Array::from_parts(Rc::new(buffer), 0, shape, data_type) }+ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }}}diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex 363d77d86..fed9629f1 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -4,13 +4,13 @@ pub trait CommandBuffer {type Backend: Backend;fn with_compute_encoder<T>(- &self,- callback: impl FnOnce(&<Self::Backend as Backend>::ComputeEncoder) -> T,+ &mut self,+ callback: impl FnOnce(&mut <Self::Backend as Backend>::ComputeEncoder) -> T,) -> T;fn with_copy_encoder<T>(- &self,- callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,+ &mut self,+ callback: impl FnOnce(&mut <Self::Backend as Backend>::CopyEncoder) -> T,) -> T;fn encode_wait_for_event(diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rsindex 9b39ac447..93c319ae2 100644--- a/crates/uzu/src/backends/common/copy_encoder.rs+++ b/crates/uzu/src/backends/common/copy_encoder.rs@@ -6,15 +6,15 @@ pub trait CopyEncoder {type Backend: Backend;fn encode_copy(- &self,+ &mut self,src: &<Self::Backend as Backend>::NativeBuffer,- dst: &<Self::Backend as Backend>::NativeBuffer,+ dst: &mut <Self::Backend as Backend>::NativeBuffer,size: usize,);fn encode_fill(- &self,- dst: &<Self::Backend as Backend>::NativeBuffer,+ &mut self,+ dst: &mut <Self::Backend as Backend>::NativeBuffer,range: Range<usize>,value: u8,);diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsindex bb89ea6e8..839b637ea 100644--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -1,4 +1,4 @@-use std::{mem::size_of, rc::Rc};+use std::{cell::RefCell, mem::size_of, ops::Deref, rc::Rc};use thiserror::Error;@@ -8,9 +8,9 @@ use crate::{};pub struct KVLayerData<B: Backend> {- pub key_buffer: Rc<B::NativeBuffer>,+ pub key_buffer: Rc<RefCell<B::NativeBuffer>>,pub key_shape: [usize; 3],- pub value_buffer: Rc<B::NativeBuffer>,+ pub value_buffer: Rc<RefCell<B::NativeBuffer>>,pub value_shape: [usize; 3],}@@ -57,7 +57,7 @@ impl<B: Backend> KVCacheUpdate<B> {in_place_data: &[KVLayerData<B>],source_indices: &[usize],destination_indices: &[usize],- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) -> Result<(), KVCacheUpdateError<B>> {command_buffer.with_compute_encoder(|encoder| {self.encode_with_encoder(in_place_data, source_indices, destination_indices, encoder)@@ -92,8 +92,8 @@ impl<B: Backend> KVCacheUpdate<B> {// non-inline is not supported yet (and is broken anyways due to a data race)for swaps_chunk in swaps.chunks(max_inline_swaps) {self.kernel.encode(- layer_data.key_buffer.as_ref(),- layer_data.value_buffer.as_ref(),+ layer_data.key_buffer.borrow().deref(),+ layer_data.value_buffer.borrow().deref(),swaps_chunk,swaps_chunk.len() as u32,num_heads as u32,diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rsindex cc408d358..70ca68396 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs@@ -82,7 +82,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {pub fn encode(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: MoeExpertsSingleDecodeArguments<B>,) {if args.k == 0 {diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rsindex 9650dbe5d..a366d82cd 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs@@ -1,3 +1,5 @@+use std::ops::Deref;+use super::{MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,@@ -45,7 +47,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {pub fn encode(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoeExpertsTwoPassArguments<B>,) {if args.total_rows == 0 {@@ -112,7 +114,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {args.x_perm_buffer,args.expert_offsets,args.w13_all,- args.hidden_buffer,+ args.hidden_buffer.deref(),args.up_biases,args.d_model as u32,args.d_ff as u32,@@ -132,7 +134,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {command_buffer.with_compute_encoder(|encoder| {let pass_b_kernel = &self.fused_down[dtype_idx];pass_b_kernel.encode(- args.hidden_buffer,+ args.hidden_buffer.deref(),args.row_expert_map,args.w2_all,args.down_biases,diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rsindex c82d14345..4d0584565 100644--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs@@ -1,3 +1,5 @@+use std::ops::Deref;+use super::{MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,};@@ -46,8 +48,8 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {pub fn encode(&self,- command_buffer: &B::CommandBuffer,- args: &MoeExpertsTwoPassArguments<B>,+ command_buffer: &mut B::CommandBuffer,+ args: &mut MoeExpertsTwoPassArguments<B>,) {if args.total_rows == 0 {return;@@ -116,7 +118,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {args.expert_offsets,args.w13_all,args.up_biases,- args.hidden_buffer,+ args.hidden_buffer.deref(),args.d_model as u32,args.d_ff as u32,args.e as u32,@@ -141,7 +143,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {command_buffer.with_compute_encoder(|encoder| {let kernel_pass_b = &self.pass_b_indirect[dtype_idx];kernel_pass_b.encode(- args.hidden_buffer,+ args.hidden_buffer.deref(),args.expert_offsets,args.w2_all,args.down_biases,@@ -162,7 +164,7 @@ pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {pub x_perm_buffer: &'a B::NativeBuffer,pub expert_offsets: &'a B::NativeBuffer,pub row_expert_map: &'a B::NativeBuffer,- pub hidden_buffer: &'a B::NativeBuffer,+ pub hidden_buffer: &'a mut B::NativeBuffer,pub output_buffer: &'a B::NativeBuffer,pub w13_all: &'a B::NativeBuffer,pub w2_all: &'a B::NativeBuffer,diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rsindex bd4bc07bb..b812598d0 100644--- a/crates/uzu/src/backends/common/kernel/moe/gather.rs+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs@@ -34,7 +34,7 @@ impl<B: Backend> MoeGatherKernels<B> {pub fn encode(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,dtype: DataType,args: &MoeGatherArguments<B>,) {diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rsindex 3590cb360..8a4c90bd9 100644--- a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs@@ -22,7 +22,7 @@ impl<B: Backend> MoeTileMapKernels<B> {pub fn encode_counts(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoeTileCountsArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {@@ -32,7 +32,7 @@ impl<B: Backend> MoeTileMapKernels<B> {pub fn encode_scan(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoeTileScanArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {@@ -48,7 +48,7 @@ impl<B: Backend> MoeTileMapKernels<B> {pub fn encode_build_map(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoeTileMapBuildArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {@@ -65,7 +65,7 @@ impl<B: Backend> MoeTileMapKernels<B> {pub fn encode_dispatch_args(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoeTileDispatchArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rsindex 43548c713..106a9f657 100644--- a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs@@ -68,7 +68,7 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_counts(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoePassATileCountsArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {@@ -78,7 +78,7 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_scan(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoePassATileScanArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {@@ -88,7 +88,7 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_row_map(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoePassARowMapArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {@@ -104,7 +104,7 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_build_map(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoePassATileBuildArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {@@ -122,7 +122,7 @@ impl<B: Backend> MoePassATileKernels<B> {pub fn encode_dispatch_args(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,args: &MoePassATileDispatchArguments<B>,) {command_buffer.with_compute_encoder(|encoder| {diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rsindex fc5d832ee..c8e37bb01 100644--- a/crates/uzu/src/backends/metal/backend.rs+++ b/crates/uzu/src/backends/metal/backend.rs@@ -11,8 +11,8 @@ impl Backend for Metal {type Context = MetalContext;type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;- type ComputeEncoder = ProtocolObject<dyn MTLComputeCommandEncoder>;- type CopyEncoder = ProtocolObject<dyn MTLBlitCommandEncoder>;+ type ComputeEncoder = Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>;+ type CopyEncoder = Retained<ProtocolObject<dyn MTLBlitCommandEncoder>>;type Event = Retained<ProtocolObject<dyn MTLEvent>>;type Kernels = MetalKernels;type Error = MetalError;diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex ab89ccd33..278300aa9 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -10,12 +10,12 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {type Backend = Metal;fn with_compute_encoder<T>(- &self,- callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,+ &mut self,+ callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,) -> T {- let encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");+ let mut encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");- let ret = callback(&encoder);+ let ret = callback(&mut encoder);encoder.end_encoding();@@ -23,12 +23,12 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {}fn with_copy_encoder<T>(- &self,- callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,+ &mut self,+ callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,) -> T {- let encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");+ let mut encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");- let ret = callback(&encoder);+ let ret = callback(&mut encoder);encoder.end_encoding();diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rsindex 12cf03bff..e74776e15 100644--- a/crates/uzu/src/backends/metal/copy_encoder.rs+++ b/crates/uzu/src/backends/metal/copy_encoder.rs@@ -1,18 +1,18 @@use std::ops::Range;use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer};-use objc2::runtime::ProtocolObject;+use objc2::{rc::Retained, runtime::ProtocolObject};use super::Metal;use crate::backends::common::{Backend, CopyEncoder};-impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {+impl CopyEncoder for Retained<ProtocolObject<dyn MTLBlitCommandEncoder>> {type Backend = Metal;fn encode_copy(- &self,+ &mut self,src: &<Self::Backend as Backend>::NativeBuffer,- dst: &<Self::Backend as Backend>::NativeBuffer,+ dst: &mut <Self::Backend as Backend>::NativeBuffer,size: usize,) {assert!(src.length() >= size && dst.length() >= size);@@ -21,8 +21,8 @@ impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {}fn encode_fill(- &self,- dst: &<Self::Backend as Backend>::NativeBuffer,+ &mut self,+ dst: &mut <Self::Backend as Backend>::NativeBuffer,range: Range<usize>,value: u8,) {diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 941c994b3..479d205f7 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -111,35 +111,35 @@ wherelet encoding_params = EncodingParameters::new(false, true, false);- self.context.embed.encode(&mut state, &encoding_params, &self.context.command_buffer);- self.context.embedding_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);+ self.context.embed.encode(&mut state, &encoding_params, &mut self.context.command_buffer);+ self.context.embedding_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);#[cfg(feature = "tracing")]{let traces = state.traces().clone();state.encode_copy_array(- &self.context.command_buffer,+ &mut self.context.command_buffer,ArrayId::Main,traces.borrow().embedding_norm().clone(),);}for layer in self.context.layers.iter() {- layer.encode(&mut state, &encoding_params, &self.context.command_buffer);+ layer.encode(&mut state, &encoding_params, &mut self.context.command_buffer);}- self.context.output_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);+ self.context.output_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);#[cfg(feature = "tracing")]{let traces = state.traces().clone();state.encode_copy_array(- &self.context.command_buffer,+ &mut self.context.command_buffer,ArrayId::Main,traces.borrow().output_norm.clone(),);}- self.context.pooling.encode(&mut state, &encoding_params, &self.context.command_buffer);+ self.context.pooling.encode(&mut state, &encoding_params, &mut self.context.command_buffer);- self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);+ self.context.prediction_head.encode(&mut state, &encoding_params, &mut self.context.command_buffer);self.context.command_buffer.submit();self.context.command_buffer.wait_until_completed();@@ -174,14 +174,14 @@ wherelet encoding_params = EncodingParameters::new(false, true, false);- self.context.embed.encode(&mut state, &encoding_params, &self.context.command_buffer);- self.context.embedding_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);+ self.context.embed.encode(&mut state, &encoding_params, &mut self.context.command_buffer);+ self.context.embedding_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);for layer in self.context.layers.iter() {- layer.encode(&mut state, &encoding_params, &self.context.command_buffer);+ layer.encode(&mut state, &encoding_params, &mut self.context.command_buffer);}- self.context.output_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);- self.context.pooling.encode(&mut state, &encoding_params, &self.context.command_buffer);- self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);+ self.context.output_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);+ self.context.pooling.encode(&mut state, &encoding_params, &mut self.context.command_buffer);+ self.context.prediction_head.encode(&mut state, &encoding_params, &mut self.context.command_buffer);self.context.command_buffer.submit();self.context.command_buffer.wait_until_completed();diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rsindex dd09a8315..133866eaa 100644--- a/crates/uzu/src/encodable_block/activation.rs+++ b/crates/uzu/src/encodable_block/activation.rs@@ -1,5 +1,7 @@//! Activation encodable.+use std::ops::Deref;+use super::{EncodableBlock, EncodingParameters};use crate::{DataType,@@ -60,6 +62,12 @@ impl<B: Backend> EncodableBlock<B> for Activation<B> {ActivationConfig::Identity => panic!("Identity activation is not supported for kernel"),};- self.kernel.encode(input_array.buffer(), output_array.buffer(), n as u32, act_type, encoder);+ self.kernel.encode(+ input_array.buffer().borrow().deref(),+ output_array.buffer().borrow().deref(),+ n as u32,+ act_type,+ encoder,+ );}}diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex 39859aceb..444127f32 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -1,6 +1,6 @@//! Attention kernel encodable.-use std::collections::HashMap;+use std::{collections::HashMap, ops::Deref};use crate::{DataType,@@ -132,7 +132,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, &encoder));@@ -221,32 +221,39 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {};let rotated_keys_array = rotated_keys_binding[0].borrow_mut();- let rotated_keys_buffer = rotated_keys_array.buffer();+ let rotated_keys_buf_rc = rotated_keys_array.buffer();+ let rotated_keys_buf_borrow = rotated_keys_buf_rc.borrow();let qkv_array = qkv_binding[0].borrow_mut();- let qkv_buffer = qkv_array.buffer();+ let qkv_buf_rc = qkv_array.buffer();+ let qkv_buf_borrow = qkv_buf_rc.borrow();// Get KV cache buffers only if KV cache exists (LLM mode)let has_kv_cache = state.cache_layers().is_some();let key_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Keys(self.layer_index)]));let key_cache_array = key_cache_binding.as_ref().map(|b| b[0].borrow_mut());+ let key_cache_buf_rc = key_cache_array.as_ref().map(|a| a.buffer());+ let key_cache_buf_borrow = key_cache_buf_rc.as_ref().map(|rc| rc.borrow());let value_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Values(self.layer_index)]));let value_cache_array = value_cache_binding.as_ref().map(|b| b[0].borrow_mut());+ let value_cache_buf_rc = value_cache_array.as_ref().map(|a| a.buffer());+ let value_cache_buf_borrow = value_cache_buf_rc.as_ref().map(|rc| rc.borrow());let extracted_values_binding = (!has_kv_cache).then(|| state.arrays(&[ArrayId::ExtractedValues]));let extracted_values_array = extracted_values_binding.as_ref().map(|b| b[0].borrow_mut());+ let extracted_values_buf_rc = extracted_values_array.as_ref().map(|a| a.buffer());+ let extracted_values_buf_borrow = extracted_values_buf_rc.as_ref().map(|rc| rc.borrow());// For classifiers (no KV cache): extract values from QKV into a dedicated extracted_values buffer.if !has_kv_cache {- let extracted_values_buf = extracted_values_array.as_ref().unwrap().buffer();self.update_kv_cache_kernel.encode(- rotated_keys_buffer,- qkv_buffer,+ rotated_keys_buf_borrow.deref(),+ qkv_buf_borrow.deref(),// keys already in desired layout; harmless overwrite- rotated_keys_buffer,- extracted_values_buf,+ rotated_keys_buf_borrow.deref(),+ extracted_values_buf_borrow.as_ref().unwrap().deref(),num_groups as u32,num_heads as u32,head_dim as u32,@@ -257,22 +264,24 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {);}- let key_cache_buffer = if has_kv_cache {- key_cache_array.as_ref().unwrap().buffer()+ let key_cache_buffer: &B::NativeBuffer = if has_kv_cache {+ key_cache_buf_borrow.as_ref().unwrap().deref()} else {- rotated_keys_buffer+ rotated_keys_buf_borrow.deref()};- let value_cache_buffer = if has_kv_cache {- value_cache_array.as_ref().unwrap().buffer()+ let value_cache_buffer: &B::NativeBuffer = if has_kv_cache {+ value_cache_buf_borrow.as_ref().unwrap().deref()} else {- extracted_values_array.as_ref().unwrap().buffer()+ extracted_values_buf_borrow.as_ref().unwrap().deref()};let queries_array = rotated_queries_binding[0].borrow_mut();- let queries_buffer = queries_array.buffer();+ let queries_buf_rc = queries_array.buffer();+ let queries_buf_borrow = queries_buf_rc.borrow();let attention_output_array = attention_output_binding[0].borrow_mut();- let attention_output_buffer = attention_output_array.buffer();+ let attention_output_buf_rc = attention_output_array.buffer();+ let attention_output_buf_borrow = attention_output_buf_rc.borrow();let attention_bias_array_borrow = if use_mask {Some(@@ -284,29 +293,36 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {} else {None};- let attention_bias_buffer = attention_bias_array_borrow.as_ref().map(|a| a.buffer());+ let attention_bias_buf_rc = attention_bias_array_borrow.as_ref().map(|a| a.buffer());+ let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());+ let attention_bias_buffer: Option<&B::NativeBuffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);let sums_binding = state.arrays(&[ArrayId::AttentionSums]);let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);let partials_array = partials_binding[0].borrow_mut();- let partials_buffer = partials_array.buffer();+ let partials_buf_rc = partials_array.buffer();+ let partials_buf_borrow = partials_buf_rc.borrow();let sums_array = sums_binding[0].borrow_mut();- let sums_buffer = sums_array.buffer();+ let sums_buf_rc = sums_array.buffer();+ let sums_buf_borrow = sums_buf_rc.borrow();let maxs_array = maxs_binding[0].borrow_mut();- let maxs_buffer = maxs_array.buffer();+ let maxs_buf_rc = maxs_array.buffer();+ let maxs_buf_borrow = maxs_buf_rc.borrow();let sinks_borrow = sinks_binding.as_ref().map(|binding| binding[0].borrow());- let sinks_buffer = sinks_borrow.as_ref().map(|b| b.buffer());+ let sinks_buf_rc = sinks_borrow.as_ref().map(|b| b.buffer());+ let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());+ let sinks_buffer: Option<&B::NativeBuffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());// Only update KV cache for LLM mode (not for classifiers)if has_kv_cache {self.update_kv_cache_kernel.encode(- rotated_keys_buffer,- qkv_buffer,+ rotated_keys_buf_borrow.deref(),+ qkv_buf_borrow.deref(),key_cache_buffer,value_cache_buffer,num_groups as u32,@@ -341,10 +357,10 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {match variant {KernelVariant::Gemm => {let args = AttentionGemmArguments {- queries_buffer,+ queries_buffer: queries_buf_borrow.deref(),keys_buffer: key_cache_buffer,values_buffer: value_cache_buffer,- output_buffer: attention_output_buffer,+ output_buffer: attention_output_buf_borrow.deref(),mask_buffer: attention_bias_buffer,sinks_buffer,num_heads,@@ -367,10 +383,10 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),};kernel.encode(- queries_buffer,+ queries_buf_borrow.deref(),key_cache_buffer,value_cache_buffer,- attention_output_buffer,+ attention_output_buf_borrow.deref(),gqa_factor as u32,sequence_length as u32,k_head_stride as u32,@@ -398,12 +414,12 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),};kernel_pass1.encode(- queries_buffer,+ queries_buf_borrow.deref(),key_cache_buffer,value_cache_buffer,- partials_buffer,- sums_buffer,- maxs_buffer,+ partials_buf_borrow.deref(),+ sums_buf_borrow.deref(),+ maxs_buf_borrow.deref(),gqa_factor as u32,sequence_length as u32,k_head_stride as u32,@@ -421,10 +437,10 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {&compute_encoder,);kernel_pass2.encode(- partials_buffer,- sums_buffer,- maxs_buffer,- attention_output_buffer,+ partials_buf_borrow.deref(),+ sums_buf_borrow.deref(),+ maxs_buf_borrow.deref(),+ attention_output_buf_borrow.deref(),num_heads as u32,suffix_length as u32,&compute_encoder,diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex 9a5d7cd9c..a452cd3ed 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -253,7 +253,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {#[cfg(not(feature = "tracing"))]{diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex 3f0f365b2..39c8e0f3d 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -178,7 +178,7 @@ impl<B: Backend> EncodableBlock<B> for Decoder<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {self.embed.encode(state, parameters, command_buffer);diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rsindex 544532612..910273e7e 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs@@ -1,6 +1,6 @@//! Full precision embedding lookup encodable.-use std::rc::Rc;+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -38,7 +38,7 @@ pub enum FullPrecisionEmbeddingLookupError<B: Backend> {pub struct FullPrecisionEmbeddingLookup<B: Backend> {kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,- weights_buffer: Rc<B::NativeBuffer>,+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,vocab_size: u32,model_dim: u32,input_scale: f32,@@ -78,7 +78,7 @@ impl<B: Backend> FullPrecisionEmbeddingLookup<B> {Ok(Self {kernel,- weights_buffer: weights.buffer_rc(),+ weights_buffer: weights.buffer(),vocab_size: vocab_size as u32,model_dim: model_dim as u32,input_scale: input_scale.unwrap_or(1.0),@@ -103,9 +103,9 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {let output_array = arrays[1].borrow_mut();self.kernel.encode(- token_ids_array.buffer(),- self.weights_buffer.as_ref(),- output_array.buffer(),+ token_ids_array.buffer().borrow().deref(),+ self.weights_buffer.borrow().deref(),+ output_array.buffer().borrow().deref(),batch_size as u32,self.vocab_size,self.model_dim,diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rsindex 8702efb06..7262b584a 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs@@ -1,4 +1,4 @@-use std::{cell::RefCell, rc::Rc};+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -41,7 +41,7 @@ whereB::Kernels: MatmulKernels,{kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,- weights_buffer: Rc<B::NativeBuffer>,+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,vocab_size: usize,model_dim: usize,}@@ -89,7 +89,7 @@ whereOk(Self {kernel: RefCell::new(kernel),- weights_buffer: weights.buffer_rc(),+ weights_buffer: weights.buffer(),vocab_size,model_dim,})@@ -125,10 +125,10 @@ wherestate.context(),encoder,FullPrecisionMatmulArguments {- a: input_array.buffer(),+ a: input_array.buffer().borrow().deref(),a_offset: sampling_start * self.model_dim * element_size,- b: &self.weights_buffer,- output: output_array.buffer(),+ b: self.weights_buffer.borrow().deref(),+ output: output_array.buffer().borrow().deref(),bias: None,batch: batch_size,input_dim: self.model_dim,diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rsindex be9deebe3..fd9c37509 100644--- a/crates/uzu/src/encodable_block/full_precision_linear.rs+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs@@ -1,4 +1,4 @@-use std::{cell::RefCell, rc::Rc};+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -49,8 +49,8 @@ whereB::Kernels: MatmulKernels,{kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,- bias_buffer: Option<Rc<B::NativeBuffer>>,- weights_buffer: Rc<B::NativeBuffer>,+ bias_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,input_dim: usize,output_dim: usize,input_array_id: ArrayId,@@ -108,7 +108,7 @@ where});}- Some(biases.buffer_rc())+ Some(biases.buffer())},Err(_) => None,};@@ -119,7 +119,7 @@ whereOk(Self {kernel: RefCell::new(kernel),bias_buffer,- weights_buffer: weights.buffer_rc(),+ weights_buffer: weights.buffer(),input_dim,output_dim,input_array_id,@@ -147,15 +147,16 @@ wherelet input_array = arrays[0].borrow_mut();let output_array = arrays[1].borrow_mut();+ let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());self.kernel.borrow_mut().encode(state.context(),encoder,FullPrecisionMatmulArguments {- a: input_array.buffer(),+ a: input_array.buffer().borrow().deref(),a_offset: 0,- b: &self.weights_buffer,- output: output_array.buffer(),- bias: self.bias_buffer.as_ref().map(|b| b.as_ref()),+ b: self.weights_buffer.borrow().deref(),+ output: output_array.buffer().borrow().deref(),+ bias: bias_borrow.as_deref(),batch: batch_size,input_dim: self.input_dim,output_dim: self.output_dim,diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex dfca55db8..acb6db004 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -264,7 +264,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {// In non-tracing builds, if every sub-block supports shared encoding,// we can run the entire layer in a single compute encoder.diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rsindex 866fe03c5..3992419bb 100644--- a/crates/uzu/src/encodable_block/layer_norm.rs+++ b/crates/uzu/src/encodable_block/layer_norm.rs@@ -1,6 +1,6 @@//! LayerNorm encodable.-use std::rc::Rc;+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -29,7 +29,7 @@ pub struct LayerNorm<B: Backend> {config: NormalizationConfig,input_array_id: ArrayId,output_array_id: ArrayId,- scales_buffer: Rc<B::NativeBuffer>,+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,}impl<B: Backend> LayerNorm<B> {@@ -60,7 +60,7 @@ impl<B: Backend> LayerNorm<B> {config,input_array_id,output_array_id,- scales_buffer: scales.buffer_rc(),+ scales_buffer: scales.buffer(),})}}@@ -96,9 +96,9 @@ impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {};self.kernel.encode(- input_array.buffer(),- self.scales_buffer.as_ref(),- output_array.buffer(),+ input_array.buffer().borrow().deref(),+ self.scales_buffer.borrow().deref(),+ output_array.buffer().borrow().deref(),batch_size,model_dim,self.config.epsilon,diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex 45ec30fab..c248bebce 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -1,6 +1,6 @@//! Mamba2 SSM mixer encodable.-use std::env;+use std::{env, ops::Deref};use crate::{Activation, DataType,@@ -177,29 +177,23 @@ impl<B: Backend> MambaMixer<B> {]);let in_proj = arrays[0].borrow();- let input_buf = in_proj.buffer();-let conv_inputs = arrays[1].borrow();- let conv_buf = conv_inputs.buffer();-let gate = arrays[2].borrow();- let gate_buf = gate.buffer();-let dt = arrays[3].borrow();- let dt_buf = dt.buffer();- let bias_buf = self.gate_bias.buffer();+ let bias_buf_rc = self.gate_bias.buffer();+ let bias_buf_borrow = bias_buf_rc.borrow();let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let num_heads = self.config.num_heads;let total_dim = conv_dim + inner_dim + num_heads;self.split_inproj.encode(- input_buf,- conv_buf,- gate_buf,- dt_buf,- bias_buf,+ in_proj.buffer().borrow().deref(),+ conv_inputs.buffer().borrow().deref(),+ gate.buffer().borrow().deref(),+ dt.buffer().borrow().deref(),+ bias_buf_borrow.deref(),suffix_length as u32,total_dim as u32,conv_dim as u32,@@ -228,14 +222,10 @@ impl<B: Backend> MambaMixer<B> {let b_arr = arrays[3].borrow();let c_arr = arrays[4].borrow();- let input_buf = conv_inputs.buffer();- let state_buf = conv_state.buffer();- let x_buf = x_arr.buffer();- let b_buf = b_arr.buffer();- let c_buf = c_arr.buffer();-- let weight_buf = self.conv_weight.buffer();- let bias_buf = self.conv_bias.as_ref().map(|arr| arr.buffer());+ let weight_buf_rc = self.conv_weight.buffer();+ let weight_buf_borrow = weight_buf_rc.borrow();+ let bias_buf_rc = self.conv_bias.as_ref().map(|arr| arr.buffer());+ let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();@@ -245,14 +235,14 @@ impl<B: Backend> MambaMixer<B> {if suffix_length == 1 {if conv_dim > 0 && self.config.kernel_size > 0 {self.conv_decode.encode(- input_buf,- weight_buf,- bias_buf,- state_buf,- x_buf,- b_buf,- c_buf,- state_buf,+ conv_inputs.buffer().borrow().deref(),+ weight_buf_borrow.deref(),+ bias_buf_borrow.as_deref(),+ conv_state.buffer().borrow().deref(),+ x_arr.buffer().borrow().deref(),+ b_arr.buffer().borrow().deref(),+ c_arr.buffer().borrow().deref(),+ conv_state.buffer().borrow().deref(),self.config.kernel_size as u32,conv_dim as u32,state_stride as u32,@@ -266,12 +256,12 @@ impl<B: Backend> MambaMixer<B> {} else {let padded_buf = if state_stride > 0 {let array = state.conv_padded_buffer().expect("Missing conv padded buffer");- let buffer = array.borrow().buffer_rc();+ let buffer = array.borrow().buffer();self.conv_pack.encode(- state_buf,- input_buf,- buffer.as_ref(),+ conv_state.buffer().borrow().deref(),+ conv_inputs.buffer().borrow().deref(),+ buffer.borrow().deref(),state_stride as u32,conv_dim as u32,suffix_length as u32,@@ -285,15 +275,18 @@ impl<B: Backend> MambaMixer<B> {};if conv_dim > 0 && self.config.kernel_size > 0 {- let conv_source = padded_buf.as_ref().map(|b| b.as_ref()).unwrap_or(input_buf);+ let padded_borrow = padded_buf.as_ref().map(|b| b.borrow());+ let conv_inputs_borrow = conv_inputs.buffer();+ let conv_inputs_ref = conv_inputs_borrow.borrow();+ let conv_source: &B::NativeBuffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());self.conv_scan.encode(conv_source,- weight_buf,- bias_buf,- x_buf,- b_buf,- c_buf,- state_buf,+ weight_buf_borrow.deref(),+ bias_buf_borrow.as_deref(),+ x_arr.buffer().borrow().deref(),+ b_arr.buffer().borrow().deref(),+ c_arr.buffer().borrow().deref(),+ conv_state.buffer().borrow().deref(),suffix_length as u32,self.config.kernel_size as u32,conv_dim as u32,@@ -324,38 +317,26 @@ impl<B: Backend> MambaMixer<B> {]);let x = base_arrays[0].borrow();- let x_buf = x.buffer();-let b = base_arrays[1].borrow();- let b_buf = b.buffer();-let c = base_arrays[2].borrow();- let c_buf = c.buffer();-let dt = base_arrays[3].borrow();- let dt_buf = dt.buffer();-let z = base_arrays[4].borrow();- let z_buf = z.buffer();-- let state_buf = base_arrays[5].borrow();- let state_raw = state_buf.buffer();-+ let state_arr = base_arrays[5].borrow();let out = base_arrays[6].borrow();- let out_buf = out.buffer();- let skip = self.skip_connection_weight.buffer();+ let skip_rc = self.skip_connection_weight.buffer();+ let skip_borrow = skip_rc.borrow();self.ssd_prefill.encode(encoder,SSDPrefillArguments {- x: x_buf,- dt: dt_buf,- b: b_buf,- c: c_buf,- d: skip,- z: z_buf,- state: state_raw,- y: out_buf,+ x: x.buffer().borrow().deref(),+ dt: dt.buffer().borrow().deref(),+ b: b.buffer().borrow().deref(),+ c: c.buffer().borrow().deref(),+ d: skip_borrow.deref(),+ z: z.buffer().borrow().deref(),+ state: state_arr.buffer().borrow().deref(),+ y: out.buffer().borrow().deref(),suffix_len: suffix_length,group_size: (self.config.num_heads / self.config.num_groups) as i32,state_size: self.config.state_dim as i32,@@ -386,26 +367,15 @@ impl<B: Backend> MambaMixer<B> {ArrayId::AttentionOutput,]);let x = arrays[0].borrow();- let x_buf = x.buffer();-let b = arrays[1].borrow();- let b_buf = b.buffer();-let c = arrays[2].borrow();- let c_buf = c.buffer();-let dt = arrays[3].borrow();- let dt_buf = dt.buffer();-let z = arrays[4].borrow();- let z_buf = z.buffer();-let state_arr = arrays[5].borrow();- let state_buf = state_arr.buffer();-let y = arrays[6].borrow();- let y_buf = y.buffer();- let skip_buf = self.skip_connection_weight.buffer();++ let skip_rc = self.skip_connection_weight.buffer();+ let skip_borrow = skip_rc.borrow();let h = self.config.num_heads as u32;let g = self.config.num_groups as u32;@@ -420,15 +390,15 @@ impl<B: Backend> MambaMixer<B> {let state_size = n as i32;self.ssd_update.encode(- x_buf,- dt_buf,- b_buf,- c_buf,- skip_buf,- z_buf,- state_buf,- y_buf,- state_buf,+ x.buffer().borrow().deref(),+ dt.buffer().borrow().deref(),+ b.buffer().borrow().deref(),+ c.buffer().borrow().deref(),+ skip_borrow.deref(),+ z.buffer().borrow().deref(),+ state_arr.buffer().borrow().deref(),+ y.buffer().borrow().deref(),+ state_arr.buffer().borrow().deref(),group_size as u32,state_size as u32,x_strides.as_slice(),@@ -469,7 +439,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {if self.supports_shared_encoder() {command_bufferdiff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rsindex 9a2af53a4..17bf482de 100644--- a/crates/uzu/src/encodable_block/mlp.rs+++ b/crates/uzu/src/encodable_block/mlp.rs@@ -1,5 +1,7 @@//! MLP block encodable.+use std::ops::Deref;+use crate::{backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},encodable_block::{EncodableBlock, EncodingParameters},@@ -31,7 +33,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {&self,state: &mut ForwardPassState<B>,params: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {if self.supports_shared_encoder() {command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, params, encoder));@@ -45,11 +47,13 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {let fused = arrays[0].borrow_mut();let hidden = arrays[1].borrow_mut();let m = fused.shape()[0] as i32;- let fused_buf = fused.buffer();- let hidden_buf = hidden.buffer();+ let fused_buf_rc = fused.buffer();+ let fused_buf_borrow = fused_buf_rc.borrow();+ let hidden_buf_rc = hidden.buffer();+ let hidden_buf_borrow = hidden_buf_rc.borrow();self.gate- .encode(encoder, fused_buf, hidden_buf, m)+ .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref(), m).expect("Failed to encode MLP activation/mul kernel");});@@ -81,9 +85,13 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {let fused = arrays[0].borrow_mut();let hidden = arrays[1].borrow_mut();let m = fused.shape()[0] as i32;- let fused_buf = fused.buffer();- let hidden_buf = hidden.buffer();- self.gate.encode(encoder, fused_buf, hidden_buf, m).expect("Failed to encode MLP activation/mul kernel");+ let fused_buf_rc = fused.buffer();+ let fused_buf_borrow = fused_buf_rc.borrow();+ let hidden_buf_rc = hidden.buffer();+ let hidden_buf_borrow = hidden_buf_rc.borrow();+ self.gate+ .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref(), m)+ .expect("Failed to encode MLP activation/mul kernel");drop(fused);drop(hidden);diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 15bcfa7d9..d1ae53711 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -118,7 +118,7 @@ pub trait EncodableBlock<B: Backend> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));}diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rsindex 61243d68f..61a4636ae 100644--- a/crates/uzu/src/encodable_block/moe_block.rs+++ b/crates/uzu/src/encodable_block/moe_block.rs@@ -1,6 +1,10 @@//! MoE (Mixture of Experts) block encodable.-use std::{cell::RefCell, rc::Rc};+use std::{+ cell::RefCell,+ ops::{Deref, DerefMut},+ rc::Rc,+};use thiserror::Error;@@ -24,16 +28,16 @@ use crate::{};struct RouterBlock<B: Backend> {- weights_buf: Rc<B::NativeBuffer>,- biases_buf: Rc<B::NativeBuffer>,+ weights_buf: Rc<RefCell<B::NativeBuffer>>,+ biases_buf: Rc<RefCell<B::NativeBuffer>>,}#[derive(Clone)]struct SharedMoeWeights<B: Backend> {- pub w13_buf: Rc<B::NativeBuffer>,- pub w2_buf: Rc<B::NativeBuffer>,- pub up_biases_buf: Rc<B::NativeBuffer>,- pub down_biases_buf: Rc<B::NativeBuffer>,+ pub w13_buf: Rc<RefCell<B::NativeBuffer>>,+ pub w2_buf: Rc<RefCell<B::NativeBuffer>>,+ pub up_biases_buf: Rc<RefCell<B::NativeBuffer>>,+ pub down_biases_buf: Rc<RefCell<B::NativeBuffer>>,}pub struct MoeBlock<B: Backend> {@@ -91,8 +95,8 @@ impl<B: Backend> MoeBlock<B> {let weights_arr = router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?;let biases_arr = router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?;RouterBlock {- weights_buf: weights_arr.buffer_rc(),- biases_buf: biases_arr.buffer_rc(),+ weights_buf: weights_arr.buffer(),+ biases_buf: biases_arr.buffer(),}},LinearConfig::QLoRA {@@ -146,10 +150,10 @@ impl<B: Backend> MoeBlock<B> {.map_err(MoeBlockError::ParameterLoaderError)?;let shared_weights = SharedMoeWeights {- w13_buf: w13_arr.buffer_rc(),- w2_buf: w2_arr.buffer_rc(),- up_biases_buf: up_biases_arr.buffer_rc(),- down_biases_buf: down_biases_arr.buffer_rc(),+ w13_buf: w13_arr.buffer(),+ w2_buf: w2_arr.buffer(),+ up_biases_buf: up_biases_arr.buffer(),+ down_biases_buf: down_biases_arr.buffer(),};Ok(Self {@@ -189,7 +193,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {let suffix_length = state.active_suffix_length();let arrays = state.arrays(&[@@ -215,7 +219,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {ArrayId::MoeTwoPassRowExpertMap,]);- let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<B::NativeBuffer> { array.borrow().buffer_rc() };+ let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<RefCell<B::NativeBuffer>> { array.borrow().buffer() };let mut array_iter = arrays.iter();let main_buf = clone_buffer(array_iter.next().unwrap());@@ -242,12 +246,10 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {let e = self.moe_config.num_routed_experts;let k = self.moe_config.num_active_routed_experts;-- let root = command_buffer;let k_tile = 128;// Clear internal MoE buffers- root.with_copy_encoder(|encoder| {+ command_buffer.with_copy_encoder(|encoder| {if suffix_length > 0 && k > 0 {let entries = suffix_length * k;let topk_bytes = entries * std::mem::size_of::<u32>();@@ -255,28 +257,28 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {// Clear topk_ids and tok2row buffersif topk_bytes > 0 {- encoder.encode_fill(&topk_ids_buf, 0..topk_bytes, 0xFF);+ encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);}if tok2row_bytes > 0 {- encoder.encode_fill(&tok2row_buf, 0..tok2row_bytes, 0xFF);+ encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);}// Clear hidden bufferlet hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();if hidden_bytes > 0 {- encoder.encode_fill(&hidden_buf, 0..hidden_bytes, 0);+ encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);}// Clear y_partial bufferlet y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();if y_partial_bytes > 0 {- encoder.encode_fill(&y_partial_buf, 0..y_partial_bytes, 0);+ encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);}// Clear x_perm bufferlet x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();if x_perm_bytes > 0 {- encoder.encode_fill(&x_perm_buf, 0..x_perm_bytes, 0);+ encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);}}});@@ -296,11 +298,11 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {command_buffer.with_compute_encoder(|encoder| {// Use the fused router+topk kernelself.router_topk_kernel.encode(- main_buf.as_ref(),- self.router.weights_buf.as_ref(),- self.router.biases_buf.as_ref(),- topk_ids_buf.as_ref(),- topk_probs_buf.as_ref(),+ main_buf.borrow().deref(),+ self.router.weights_buf.borrow().deref(),+ self.router.biases_buf.borrow().deref(),+ topk_ids_buf.borrow().deref(),+ topk_probs_buf.borrow().deref(),suffix_length as u32,self.model_dim as u32,e as u32,@@ -313,10 +315,10 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {command_buffer.with_compute_encoder(|encoder| {self.counts_offsets_kernel.encode(- topk_ids_buf.as_ref(),- offsets_buf.as_ref(),- sumk_buf.as_ref(),- partials_buf.as_ref(),+ topk_ids_buf.borrow().deref(),+ offsets_buf.borrow().deref(),+ sumk_buf.borrow().deref(),+ partials_buf.borrow().deref(),suffix_length as u32,e as u32,k as u32,@@ -328,9 +330,9 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {let num_tiles = ((e + 512 - 1) / 512).max(1);command_buffer.with_compute_encoder(|encoder| {self.scatter_bases_kernel.encode(- partials_buf.as_ref(),- block_bases_buf.as_ref(),- block_alloc_buf.as_ref(),+ partials_buf.borrow().deref(),+ block_bases_buf.borrow().deref(),+ block_alloc_buf.borrow().deref(),e as u32,num_blocks as u32,num_tiles as u32,@@ -340,31 +342,31 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {});command_buffer.with_compute_encoder(|encoder| {self.scatter_map_kernel.encode(- topk_ids_buf.as_ref(),- topk_probs_buf.as_ref(),- offsets_buf.as_ref(),- block_bases_buf.as_ref(),- block_alloc_buf.as_ref(),- bucketed_ids_buf.as_ref(),- bucketed_probs_buf.as_ref(),+ topk_ids_buf.borrow().deref(),+ topk_probs_buf.borrow().deref(),+ offsets_buf.borrow().deref(),+ block_bases_buf.borrow().deref(),+ block_alloc_buf.borrow().deref(),+ bucketed_ids_buf.borrow().deref(),+ bucketed_probs_buf.borrow().deref(),suffix_length as u32,e as u32,k as u32,num_blocks as u32,num_tiles as u32,- tok2row_buf.as_ref(),+ tok2row_buf.borrow().deref(),&encoder,);});self.gather_kernels.encode(- root,+ command_buffer,self.data_type,&MoeGatherArguments {- x_buffer: &main_buf,- bucketed_ids_buffer: &bucketed_ids_buf,- x_perm_buffer: &x_perm_buf,- sumk_buffer: &sumk_buf,+ x_buffer: main_buf.borrow().deref(),+ bucketed_ids_buffer: bucketed_ids_buf.borrow().deref(),+ x_perm_buffer: x_perm_buf.borrow().deref(),+ sumk_buffer: sumk_buf.borrow().deref(),t: suffix_length,k,d_model: self.model_dim,@@ -385,22 +387,22 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;self.experts_two_pass_decode_kernel.encode(- root,+ command_buffer,&MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &y_partial_buf,- w13_all: &self.shared_weights.w13_buf,- w2_all: &self.shared_weights.w2_buf,- up_biases: &self.shared_weights.up_biases_buf,- down_biases: &self.shared_weights.down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,+ x_perm_buffer: x_perm_buf.borrow().deref(),+ expert_offsets: offsets_buf.borrow().deref(),+ row_expert_map: row_expert_map_buf.borrow().deref(),+ hidden_buffer: hidden_buf.borrow_mut().deref_mut(),+ output_buffer: y_partial_buf.borrow().deref(),+ w13_all: self.shared_weights.w13_buf.borrow().deref(),+ w2_all: self.shared_weights.w2_buf.borrow().deref(),+ up_biases: self.shared_weights.up_biases_buf.borrow().deref(),+ down_biases: self.shared_weights.down_biases_buf.borrow().deref(),+ tile_counts: tile_counts_buf.borrow().deref(),+ tile_offsets: tile_offsets_buf.borrow().deref(),+ tile_map: tile_map_buf.borrow().deref(),+ total_tiles: total_tiles_buf.borrow().deref(),+ dispatch_args: dispatch_args_buf.borrow().deref(),total_rows,d_model: self.model_dim,d_ff: self.hidden_dim,@@ -419,21 +421,36 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {let total_rows = suffix_length * k;let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;- let args = MoeExpertsTwoPassArguments {- x_perm_buffer: x_perm_buf.as_ref(),- expert_offsets: offsets_buf.as_ref(),- row_expert_map: row_expert_map_buf.as_ref(),- hidden_buffer: hidden_buf.as_ref(),- output_buffer: y_partial_buf.as_ref(),- w13_all: self.shared_weights.w13_buf.as_ref(),- w2_all: self.shared_weights.w2_buf.as_ref(),- up_biases: self.shared_weights.up_biases_buf.as_ref(),- down_biases: self.shared_weights.down_biases_buf.as_ref(),- tile_counts: tile_counts_buf.as_ref(),- tile_offsets: tile_offsets_buf.as_ref(),- tile_map: tile_map_buf.as_ref(),- total_tiles: total_tiles_buf.as_ref(),- dispatch_args: dispatch_args_buf.as_ref(),+ let x_perm_borrow = x_perm_buf.borrow();+ let offsets_borrow = offsets_buf.borrow();+ let row_expert_map_borrow = row_expert_map_buf.borrow();+ let mut hidden_borrow = hidden_buf.borrow_mut();+ let y_partial_borrow = y_partial_buf.borrow();+ let w13_borrow = self.shared_weights.w13_buf.borrow();+ let w2_borrow = self.shared_weights.w2_buf.borrow();+ let up_biases_borrow = self.shared_weights.up_biases_buf.borrow();+ let down_biases_borrow = self.shared_weights.down_biases_buf.borrow();+ let tile_counts_borrow = tile_counts_buf.borrow();+ let tile_offsets_borrow = tile_offsets_buf.borrow();+ let tile_map_borrow = tile_map_buf.borrow();+ let total_tiles_borrow = total_tiles_buf.borrow();+ let dispatch_args_borrow = dispatch_args_buf.borrow();++ let mut args = MoeExpertsTwoPassArguments {+ x_perm_buffer: x_perm_borrow.deref(),+ expert_offsets: offsets_borrow.deref(),+ row_expert_map: row_expert_map_borrow.deref(),+ hidden_buffer: hidden_borrow.deref_mut(),+ output_buffer: y_partial_borrow.deref(),+ w13_all: w13_borrow.deref(),+ w2_all: w2_borrow.deref(),+ up_biases: up_biases_borrow.deref(),+ down_biases: down_biases_borrow.deref(),+ tile_counts: tile_counts_borrow.deref(),+ tile_offsets: tile_offsets_borrow.deref(),+ tile_map: tile_map_borrow.deref(),+ total_tiles: total_tiles_borrow.deref(),+ dispatch_args: dispatch_args_borrow.deref(),total_rows,d_model: self.model_dim,d_ff: self.hidden_dim,@@ -447,15 +464,15 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {silu_alpha,data_type: self.data_type,};- self.experts_two_pass_prefill_kernel.encode(root, &args);+ self.experts_two_pass_prefill_kernel.encode(command_buffer, &mut args);}command_buffer.with_compute_encoder(|encoder| {self.finalize_kernel.encode(- tok2row_buf.as_ref(),- topk_probs_buf.as_ref(),- y_partial_buf.as_ref(),- main_buf.as_ref(),+ tok2row_buf.borrow().deref(),+ topk_probs_buf.borrow().deref(),+ y_partial_buf.borrow().deref(),+ main_buf.borrow().deref(),suffix_length as u32,self.model_dim as u32,k as u32,diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rsindex 3ae7a4b14..1424c6013 100644--- a/crates/uzu/src/encodable_block/pooling.rs+++ b/crates/uzu/src/encodable_block/pooling.rs@@ -1,5 +1,7 @@//! Pooling encodable for sequence-level aggregation.+use std::ops::Deref;+use super::{EncodableBlock, EncodingParameters};use crate::{DataType,@@ -63,7 +65,7 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));@@ -92,8 +94,8 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {let pooling_array = arrays[1].borrow_mut();self.pooling_kernel.encode(- main_array.buffer(),- pooling_array.buffer(),+ main_array.buffer().borrow().deref(),+ pooling_array.buffer().borrow().deref(),seq_len as u32,self.model_dim as u32,batch_size,diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rsindex fb3b12d61..6cbca8760 100644--- a/crates/uzu/src/encodable_block/prediction_head.rs+++ b/crates/uzu/src/encodable_block/prediction_head.rs@@ -40,7 +40,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {if self.supports_shared_encoder() {command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rsindex 501930eaa..4d51f0932 100644--- a/crates/uzu/src/encodable_block/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -1,6 +1,6 @@//! QK Normalization encodable.-use std::rc::Rc;+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -30,8 +30,8 @@ pub struct QKNorm<B: Backend> {query_config: Option<NormalizationConfig>,key_config: Option<NormalizationConfig>,qkv_array_id: ArrayId,- query_scales_buffer: Option<Rc<B::NativeBuffer>>,- key_scales_buffer: Option<Rc<B::NativeBuffer>>,+ query_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,+ key_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,num_q_heads: usize,num_kv_heads: usize,head_dim: usize,@@ -75,7 +75,7 @@ impl<B: Backend> QKNorm<B> {.map_err(QKNormError::BackendError)?;query_kernel = Some(kernel);- query_scales_buffer = Some(scales.buffer_rc());+ query_scales_buffer = Some(scales.buffer());}// Setup key normalization if configured@@ -99,7 +99,7 @@ impl<B: Backend> QKNorm<B> {.map_err(QKNormError::BackendError)?;key_kernel = Some(kernel);- key_scales_buffer = Some(scales.buffer_rc());+ key_scales_buffer = Some(scales.buffer());}Ok(Self {@@ -135,7 +135,6 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {};let qkv_array = qkv_binding[0].borrow_mut();- let qkv_buffer = qkv_array.buffer();let batch_size = qkv_shape[0] as u32;// Process query normalization if configured@@ -143,9 +142,9 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {(&self.query_kernel, &self.query_scales_buffer, &self.query_config){query_kernel.encode(- qkv_buffer,- query_scales_buffer.as_ref(),- qkv_buffer,+ qkv_array.buffer().borrow().deref(),+ query_scales_buffer.borrow().deref(),+ qkv_array.buffer().borrow().deref(),batch_size,self.num_q_heads as u32,self.num_kv_heads as u32,@@ -164,9 +163,9 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {(&self.key_kernel, &self.key_scales_buffer, &self.key_config){key_kernel.encode(- qkv_buffer,- key_scales_buffer.as_ref(),- qkv_buffer,+ qkv_array.buffer().borrow().deref(),+ key_scales_buffer.borrow().deref(),+ qkv_array.buffer().borrow().deref(),batch_size,self.num_q_heads as u32,self.num_kv_heads as u32,diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rsindex dcfab22bf..7d27262f4 100644--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs+++ b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs@@ -1,6 +1,6 @@//! Quantized embedding lookup encodable.-use std::rc::Rc;+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -67,9 +67,9 @@ pub enum QuantizedEmbeddingLookupError<B: Backend> {pub struct QuantizedEmbeddingLookup<B: Backend> {kernel: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,- weights_buffer: Rc<B::NativeBuffer>,- scales_buffer: Rc<B::NativeBuffer>,- biases_buffer: Rc<B::NativeBuffer>,+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,+ biases_buffer: Rc<RefCell<B::NativeBuffer>>,mode: QuantizationMode,input_scale: f32,vocab_size: u32,@@ -198,7 +198,7 @@ impl<B: Backend> QuantizedEmbeddingLookup<B> {});}- biases.buffer_rc()+ biases.buffer()},Err(_) => {let element_size = match data_type {@@ -216,14 +216,14 @@ impl<B: Backend> QuantizedEmbeddingLookup<B> {std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);}- Rc::new(buffer)+ Rc::new(RefCell::new(buffer))},};Ok(Self {kernel,- weights_buffer: weights.buffer_rc(),- scales_buffer: scales.buffer_rc(),+ weights_buffer: weights.buffer(),+ scales_buffer: scales.buffer(),biases_buffer,mode,input_scale,@@ -257,11 +257,11 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {};self.kernel.encode(- token_ids_array.buffer(),- self.weights_buffer.as_ref(),- self.scales_buffer.as_ref(),- self.biases_buffer.as_ref(),- output_array.buffer(),+ token_ids_array.buffer().borrow().deref(),+ self.weights_buffer.borrow().deref(),+ self.scales_buffer.borrow().deref(),+ self.biases_buffer.borrow().deref(),+ output_array.buffer().borrow().deref(),batch_size as u32,self.vocab_size,self.model_dim,diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rsindex 4d13c1da8..1576d8d8c 100644--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs@@ -1,6 +1,6 @@//! Quantized embedding readout encodable.-use std::rc::Rc;+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -67,9 +67,9 @@ pub enum QuantizedEmbeddingReadoutError<B: Backend> {pub struct QuantizedEmbeddingReadout<B: Backend> {kernel: QuantizedMatmulKernelEncodable<B>,- weights_buffer: Rc<B::NativeBuffer>,- scales_buffer: Rc<B::NativeBuffer>,- biases_buffer: Rc<B::NativeBuffer>,+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,+ biases_buffer: Rc<RefCell<B::NativeBuffer>>,vocab_size: usize,model_dim: usize,}@@ -178,7 +178,7 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {got: deq_biases.data_type(),});}- deq_biases.buffer_rc()+ deq_biases.buffer()},Err(_) => {let element_size = match data_type {@@ -195,7 +195,7 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);}- Rc::new(buffer)+ Rc::new(RefCell::new(buffer))},};@@ -215,8 +215,8 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {Ok(Self {kernel,- weights_buffer: weights.buffer_rc(),- scales_buffer: scales.buffer_rc(),+ weights_buffer: weights.buffer(),+ scales_buffer: scales.buffer(),biases_buffer,vocab_size,model_dim,@@ -252,12 +252,12 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {.encode(encoder,QuantizedMatmulArguments {- a_buffer: input_array.buffer(),+ a_buffer: input_array.buffer().borrow().deref(),a_offset,- b_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- zero_points_or_biases_buffer: &self.biases_buffer,- output_buffer: output_array.buffer(),+ b_buffer: self.weights_buffer.borrow().deref(),+ scales_buffer: self.scales_buffer.borrow().deref(),+ zero_points_or_biases_buffer: self.biases_buffer.borrow().deref(),+ output_buffer: output_array.buffer().borrow().deref(),batch: batch_size,input_dim: self.model_dim,output_dim: self.vocab_size,diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rsindex 4cffe7a56..fa8f4fdd4 100644--- a/crates/uzu/src/encodable_block/quantized_linear.rs+++ b/crates/uzu/src/encodable_block/quantized_linear.rs@@ -1,4 +1,4 @@-use std::rc::Rc;+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -84,10 +84,10 @@ pub enum QuantizedLinearError<B: Backend> {pub struct QuantizedLinear<B: Backend> {kernel: QuantizedMatmulKernelEncodable<B>,bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,- biases_buffer: Option<Rc<B::NativeBuffer>>,- weights_buffer: Rc<B::NativeBuffer>,- scales_buffer: Rc<B::NativeBuffer>,- zero_points_or_biases_buffer: Rc<B::NativeBuffer>,+ biases_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,+ zero_points_or_biases_buffer: Rc<RefCell<B::NativeBuffer>>,quantization_type: QuantizedMatmulType,input_dim: usize,output_dim: usize,@@ -155,7 +155,7 @@ impl<B: Backend> QuantizedLinear<B> {});}- (QuantizedMatmulType::Mlx, deq_biases.buffer_rc())+ (QuantizedMatmulType::Mlx, deq_biases.buffer())},Err(_) => {let zero_points = parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;@@ -181,7 +181,7 @@ impl<B: Backend> QuantizedLinear<B> {});}- (QuantizedMatmulType::ZeroPoint, zero_points.buffer_rc())+ (QuantizedMatmulType::ZeroPoint, zero_points.buffer())},};@@ -204,7 +204,7 @@ impl<B: Backend> QuantizedLinear<B> {let bias_add_kernel = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type).map_err(QuantizedLinearError::BackendError)?;- (Some(bias_add_kernel), Some(biases.buffer_rc()))+ (Some(bias_add_kernel), Some(biases.buffer()))},Err(_) => (None, None),};@@ -227,8 +227,8 @@ impl<B: Backend> QuantizedLinear<B> {kernel,bias_add_kernel,biases_buffer,- weights_buffer: weights.buffer_rc(),- scales_buffer: scales.buffer_rc(),+ weights_buffer: weights.buffer(),+ scales_buffer: scales.buffer(),zero_points_or_biases_buffer,quantization_type,input_dim,@@ -254,18 +254,19 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {let batch_size = state.active_suffix_length();let input_array = arrays[0].borrow_mut();let output_array = arrays[1].borrow_mut();- let output_buffer = output_array.buffer();+ let output_buf_rc = output_array.buffer();+ let output_buf_borrow = output_buf_rc.borrow();self.kernel.encode(encoder,QuantizedMatmulArguments {- a_buffer: input_array.buffer(),+ a_buffer: input_array.buffer().borrow().deref(),a_offset: 0,- b_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,- output_buffer,+ b_buffer: self.weights_buffer.borrow().deref(),+ scales_buffer: self.scales_buffer.borrow().deref(),+ zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),+ output_buffer: output_buf_borrow.deref(),batch: batch_size,input_dim: self.input_dim,output_dim: self.output_dim,@@ -277,9 +278,9 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {let total_length = batch_size * self.output_dim;bias_add_kernel.encode_if(- output_buffer,- biases_buffer.as_ref(),- output_buffer,+ output_buf_borrow.deref(),+ biases_buffer.borrow().deref(),+ output_buf_borrow.deref(),self.output_dim as u32,total_length as u32,encoder,diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 1c0a18050..a1b7cea4c 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -1,6 +1,6 @@//! RMS Normalization encodable.-use std::rc::Rc;+use std::{cell::RefCell, ops::Deref, rc::Rc};use thiserror::Error;@@ -29,7 +29,7 @@ pub struct RMSNorm<B: Backend> {config: NormalizationConfig,input_array_id: ArrayId,output_array_id: ArrayId,- scales_buffer: Rc<B::NativeBuffer>,+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,use_sampling_range: bool,}@@ -66,7 +66,7 @@ impl<B: Backend> RMSNorm<B> {config,input_array_id,output_array_id,- scales_buffer: scales.buffer_rc(),+ scales_buffer: scales.buffer(),use_sampling_range: false,})}@@ -122,9 +122,9 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {let output_offset = batch_start * output_row_size_in_bytes;self.kernel.encode(- (input_array.buffer(), input_offset),- self.scales_buffer.as_ref(),- (output_array.buffer(), output_offset),+ (input_array.buffer().borrow().deref(), input_offset),+ self.scales_buffer.borrow().deref(),+ (output_array.buffer().borrow().deref(), output_offset),batch_len as u32,input_shape[1] as u32,self.config.epsilon,diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rsindex c54c13876..2cbd65be8 100644--- a/crates/uzu/src/encodable_block/rope.rs+++ b/crates/uzu/src/encodable_block/rope.rs@@ -1,5 +1,7 @@//! Rope (Rotary Position Embedding) encodable.+use std::ops::Deref;+use super::{EncodableBlock, EncodingParameters};use crate::{DataType,@@ -82,12 +84,12 @@ impl<B: Backend> EncodableBlock<B> for Rope<B> {let token_positions_offset = token_positions.offset();self.kernel.encode(- qkv.buffer(),- rope_cosines.buffer(),- rope_sines.buffer(),- (token_positions.buffer(), token_positions_offset),- rotated_queries.buffer(),- rotated_keys.buffer(),+ qkv.buffer().borrow().deref(),+ rope_cosines.buffer().borrow().deref(),+ rope_sines.buffer().borrow().deref(),+ (token_positions.buffer().borrow().deref(), token_positions_offset),+ rotated_queries.buffer().borrow().deref(),+ rotated_keys.buffer().borrow().deref(),head_dim as u32,num_heads as u32,num_groups as u32,diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex 6e1609c15..e3644d572 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -1,5 +1,7 @@//! Sampling kernel encodable.+use std::ops::Deref;+use super::{EncodableBlock, EncodingParameters};use crate::{DataType,@@ -74,15 +76,22 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {let bitmask = cell.borrow();let bitmask_row_len = bitmask.shape()[1];let bitmask_offset = bitmask.offset() + sampling_start * bitmask_row_len * std::mem::size_of::<u32>();- (Some(bitmask.buffer_rc()), bitmask_offset)+ (Some(bitmask.buffer()), bitmask_offset)});+ let bitmask_borrow = bitmask_buffer.as_ref().map(|b| b.borrow());+ let logits_buf_rc = logits.buffer();+ let logits_buf_borrow = logits_buf_rc.borrow();+ let seeds_buf_rc = seeds.buffer();+ let seeds_buf_borrow = seeds_buf_rc.borrow();+ let output_buf_rc = output_buffer_ref.buffer();+ let output_buf_borrow = output_buf_rc.borrow();if let Err(e) = self.kernel.encode_with_encoder(- logits.buffer(),- Some(seeds.buffer()),+ logits_buf_borrow.deref(),+ Some(seeds_buf_borrow.deref()),seeds_offset,- bitmask_buffer.as_ref().map(|b| b.as_ref()),+ bitmask_borrow.as_deref(),bitmask_offset,- output_buffer_ref.buffer(),+ output_buf_borrow.deref(),sampling_method,batch_size,vocab_size,diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex 6609366fe..6b79f4437 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -1,3 +1,5 @@+use std::ops::Deref;+use crate::{DataType,array::Array,@@ -111,7 +113,7 @@ impl<B: Backend> ShortConvMixer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {@@ -229,12 +231,10 @@ impl<B: Backend> ShortConvMixer<B> {let conv_state = arrays[1].borrow_mut();let out = arrays[2].borrow_mut();- let in_proj_buf = in_proj.buffer();- let state_buf = conv_state.buffer();- let out_buf = out.buffer();-- let weight_buf = self.conv_weight.buffer();- let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());+ let weight_buf_rc = self.conv_weight.buffer();+ let weight_buf_borrow = weight_buf_rc.borrow();+ let bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());+ let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);@@ -246,8 +246,8 @@ impl<B: Backend> ShortConvMixer<B> {let padded_size = padded_rows * self.model_dim * element_size;let padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");self.short_conv_pack.encode(- state_buf,- in_proj_buf,+ conv_state.buffer().borrow().deref(),+ in_proj.buffer().borrow().deref(),&padded_buf,state_stride as u32,suffix_length as u32,@@ -258,11 +258,11 @@ impl<B: Backend> ShortConvMixer<B> {self.short_conv_prefill.encode(&padded_buf,- in_proj_buf,- weight_buf,- bias_buf,- out_buf,- state_buf,+ in_proj.buffer().borrow().deref(),+ weight_buf_borrow.deref(),+ bias_buf_borrow.as_deref(),+ out.buffer().borrow().deref(),+ conv_state.buffer().borrow().deref(),suffix_length as u32,kernel_size as u32,self.model_dim as u32 * 3,@@ -309,16 +309,19 @@ impl<B: Backend> ShortConvMixer<B> {let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;let base_state_offset = conv_state.offset();let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();- let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());+ let trie_weight_buf_rc = self.conv_weight.buffer();+ let trie_weight_buf_borrow = trie_weight_buf_rc.borrow();+ let trie_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());+ let trie_bias_buf_borrow = trie_bias_buf_rc.as_ref().map(|rc| rc.borrow());self.short_conv_trie.encode(- (in_proj.buffer(), in_proj_offset),- self.conv_weight.buffer(),- bias_buf,- (conv_state.buffer(), base_state_offset),- (parents.buffer(), parents_offset),- (out.buffer(), out_offset),- (suffix_state.buffer(), suffix_state_offset),+ (in_proj.buffer().borrow().deref(), in_proj_offset),+ trie_weight_buf_borrow.deref(),+ trie_bias_buf_borrow.as_deref(),+ (conv_state.buffer().borrow().deref(), base_state_offset),+ (parents.buffer().borrow().deref(), parents_offset),+ (out.buffer().borrow().deref(), out_offset),+ (suffix_state.buffer().borrow().deref(), suffix_state_offset),trie_len as u32,kernel_size as u32,in_proj_stride as u32,@@ -344,17 +347,20 @@ impl<B: Backend> ShortConvMixer<B> {let conv_state = arrays[1].borrow_mut();let out = arrays[2].borrow_mut();- let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());+ let decode_weight_buf_rc = self.conv_weight.buffer();+ let decode_weight_buf_borrow = decode_weight_buf_rc.borrow();+ let decode_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());+ let decode_bias_buf_borrow = decode_bias_buf_rc.as_ref().map(|rc| rc.borrow());let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);self.short_conv_decode.encode(- in_proj.buffer(),- self.conv_weight.buffer(),- bias_buf,- conv_state.buffer(),- out.buffer(),- conv_state.buffer(),+ in_proj.buffer().borrow().deref(),+ decode_weight_buf_borrow.deref(),+ decode_bias_buf_borrow.as_deref(),+ conv_state.buffer().borrow().deref(),+ out.buffer().borrow().deref(),+ conv_state.buffer().borrow().deref(),suffix_length as u32,kernel_size as u32,self.model_dim as u32 * 3,@@ -370,7 +376,7 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,) {if self.supports_shared_encoder() {command_bufferdiff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rsindex da725f0b5..9aa6ba3da 100644--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs@@ -1,5 +1,7 @@//! Tensor add-swap encodable.+use std::ops::Deref;+use super::{EncodableBlock, EncodingParameters};use crate::{DataType,@@ -48,6 +50,11 @@ impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {let skip_array = arrays[0].borrow_mut();let main_array = arrays[1].borrow_mut();- self.kernel.encode(skip_array.buffer(), main_array.buffer(), length as u32, encoder);+ self.kernel.encode(+ skip_array.buffer().borrow().deref(),+ main_array.buffer().borrow().deref(),+ length as u32,+ encoder,+ );}}diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rsindex d6d230fb3..211c398a1 100644--- a/crates/uzu/src/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/encodable_block/tensor_copy.rs@@ -1,5 +1,7 @@//! Tensor copy encodable.+use std::ops::Deref;+use super::{EncodableBlock, EncodingParameters};use crate::{DataType,@@ -48,6 +50,11 @@ impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {let source_array = arrays[0].borrow_mut();let destination_array = arrays[1].borrow_mut();- self.kernel.encode(source_array.buffer(), destination_array.buffer(), length as u32, encoder);+ self.kernel.encode(+ source_array.buffer().borrow().deref(),+ destination_array.buffer().borrow().deref(),+ length as u32,+ encoder,+ );}}diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rsindex 87d8b82df..e3485cd81 100644--- a/crates/uzu/src/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -276,7 +276,7 @@ impl<B: Backend> CacheLayers<B> {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,kv_cache_update: &KVCacheUpdate<B>,) {let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rsindex 9074accdf..def9925c7 100644--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs@@ -163,7 +163,7 @@ impl<B: Backend> KVCacheLayer<B> {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,kv_cache_update: &KVCacheUpdate<B>,) {match &mut self.state {@@ -213,7 +213,7 @@ impl<B: Backend> KVCacheLayer<B> {&self,source_indices: &[usize],destination_indices: &[usize],- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,kv_cache_update: &KVCacheUpdate<B>,) {if source_indices == destination_indices {@@ -224,9 +224,9 @@ impl<B: Backend> KVCacheLayer<B> {let v_shape = self.values.borrow().shape().to_vec();let layer_data = KVLayerData {- key_buffer: self.keys.borrow().buffer_rc(),+ key_buffer: self.keys.borrow().buffer(),key_shape: [k_shape[0], k_shape[1], k_shape[2]],- value_buffer: self.values.borrow().buffer_rc(),+ value_buffer: self.values.borrow().buffer(),value_shape: [v_shape[0], v_shape[1], v_shape[2]],};diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex c61b7ea32..b78a8c777 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -1,4 +1,9 @@-use std::{cell::RefCell, collections::HashMap, rc::Rc};+use std::{+ cell::RefCell,+ collections::HashMap,+ ops::{Deref, DerefMut},+ rc::Rc,+};#[cfg(feature = "tracing")]use crate::forward_pass::traces::ActivationTrace;@@ -152,8 +157,8 @@ impl<B: Backend> ForwardPassState<B> {external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,skip_token_ids_copy: bool,should_fill_attention_bias: bool,- async_positions: Option<(Rc<B::NativeBuffer>, usize)>,- async_seeds: Option<(Rc<B::NativeBuffer>, usize)>,+ async_positions: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,+ async_seeds: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,) -> Self {let suffix_length = token_ids.len();assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");@@ -387,7 +392,9 @@ impl<B: Backend> ForwardPassState<B> {let create_buffer = |size: usize| -> ArrayCell<B> {let buffer_size = size * data_type.size_in_bytes();let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- RefCell::new(unsafe { Array::from_parts(Rc::new(buffer), 0, &[batch_size, size / batch_size], data_type) })+ RefCell::new(unsafe {+ Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, size / batch_size], data_type)+ })};ClassifierModeState {@@ -397,7 +404,9 @@ impl<B: Backend> ForwardPassState<B> {classifier_logits: {let buffer_size = batch_size * num_labels * data_type.size_in_bytes();let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- RefCell::new(unsafe { Array::from_parts(Rc::new(buffer), 0, &[batch_size, num_labels], data_type) })+ RefCell::new(unsafe {+ Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, num_labels], data_type)+ })},traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(context,@@ -423,7 +432,7 @@ impl<B: Backend> ForwardPassState<B> {let size: usize = dims.iter().product();let buffer_size = size * data_type.size_in_bytes();let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");- RefCell::new(unsafe { Array::from_parts(Rc::new(buffer), 0, dims, data_type) })+ RefCell::new(unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, dims, data_type) })};ClassifierModeState {@@ -768,7 +777,7 @@ impl<B: Backend> ForwardPassState<B> {pub fn encode_copy_array(&self,- command_buffer: &B::CommandBuffer,+ command_buffer: &mut B::CommandBuffer,source_array_id: ArrayId,destination_array: RefCell<Array<B>>,) {@@ -776,14 +785,14 @@ impl<B: Backend> ForwardPassState<B> {let src_borrow = source_ref.borrow();let dst_borrow = destination_array.borrow();- let src_buf = src_borrow.buffer();- let dst_buf = dst_borrow.buffer();+ let src_buf_rc = src_borrow.buffer();+ let dst_buf_rc = dst_borrow.buffer();let copy_size_bytes = dst_borrow.size();debug_assert_eq!(dst_borrow.size(), src_borrow.size());command_buffer.with_copy_encoder(|encoder| {- encoder.encode_copy(src_buf, dst_buf, copy_size_bytes);+ encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);});}}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 31450f93d..5cb0c89ea 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -1,4 +1,11 @@-use std::{collections::HashMap, iter::repeat_n, ops::Deref, path::Path, sync::Arc, time::Instant};+use std::{+ collections::HashMap,+ iter::repeat_n,+ ops::{Deref, DerefMut},+ path::Path,+ sync::Arc,+ time::Instant,+};use itertools::{Either, Itertools, izip};@@ -402,7 +409,7 @@ wherelet last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;let token_position = unsafe {- let ptr = async_positions_buffer.cpu_ptr().as_ptr() as *const u32;+ let ptr = async_positions_buffer.borrow().cpu_ptr().as_ptr() as *const u32;*ptr.add(pass_idx) as usize};@@ -468,30 +475,36 @@ whereself.context.executables.encode(&mut state,&EncodingParameters::new(false, false, false),- self.context.command_buffer.borrow().deref(),+ self.context.command_buffer.borrow_mut().deref_mut(),);// Encode samplingself.context.gpu_sampler.encode(&mut state,&EncodingParameters::new(false, false, false),- self.context.command_buffer.borrow().deref(),+ self.context.command_buffer.borrow_mut().deref_mut(),);// Copy sampled token: sampling_output → token_ids (for next pass)// and sampling_output → results[slot] (for callback)let sampling_output = state.sampling_output().expect("sampling_output must exist after sampling encode");let sampling_output_binding = sampling_output.borrow();- let sampling_output_buffer = sampling_output_binding.buffer();+ let sampling_output_buf_rc = sampling_output_binding.buffer();+ let sampling_output_buf_borrow = sampling_output_buf_rc.borrow();let token_ids_binding = self.context.scratch_buffers.token_ids.borrow();- let token_ids_buffer = token_ids_binding.buffer();+ let token_ids_buf_rc = token_ids_binding.buffer();+ let token_ids_buf_borrow = token_ids_buf_rc.borrow();- self.context.command_buffer.borrow().with_compute_encoder(|encoder| {- self.context.token_copy_sampled.encode(sampling_output_buffer, token_ids_buffer, encoder);+ self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {+ self.context.token_copy_sampled.encode(+ sampling_output_buf_borrow.deref(),+ token_ids_buf_borrow.deref(),+ encoder,+ );let results_offset = slot * std::mem::size_of::<u32>();self.context.token_copy_results.encode(- sampling_output_buffer,- (results_buffer.as_ref(), results_offset),+ sampling_output_buf_borrow.deref(),+ (results_buffer.borrow().deref(), results_offset),encoder,);});@@ -500,24 +513,21 @@ whereself.context.cache_layers.borrow_mut().update_after_acceptance(&[0],None,- self.context.command_buffer.borrow().deref(),+ self.context.command_buffer.borrow_mut().deref_mut(),&self.context.kv_cache_update,);self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);- self.context.command_buffer.borrow().with_compute_encoder(|encoder| {+ self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {if let Some(mask_update) = &self.context.mask_update {let updates: Vec<AttentionBiasUpdate> =self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {if let Some(update) = updates.iter().find(|u| &u.key == window_size) {if update.unmask_col >= 0 || update.mask_col >= 0 {- mask_update.encode(- mask_buffer.borrow().buffer(),- update.unmask_col,- update.mask_col,- &encoder,- );+ let mask_buf_rc = mask_buffer.borrow().buffer();+ let mask_buf_borrow = mask_buf_rc.borrow();+ mask_update.encode(mask_buf_borrow.deref(), update.unmask_col, update.mask_col, &encoder);}}}@@ -535,7 +545,7 @@ wherelet handler = move || {let token = {- let ptr = results_buffer_clone.cpu_ptr().as_ptr() as *const u32;+ let ptr = results_buffer_clone.borrow().cpu_ptr().as_ptr() as *const u32;unsafe { *ptr.add(slot) as u64 }};if let Some(cb) = callback.lock().unwrap().take() {@@ -652,7 +662,7 @@ whereself.context.gpu_sampler.encode(&mut state,&EncodingParameters::new(warmup, true, false),- &self.context.command_buffer.as_ref().borrow(),+ self.context.command_buffer.borrow_mut().deref_mut(),);}}@@ -713,14 +723,14 @@ wheresuffix_start: Option<usize>,wait_until_completed: bool,) {- let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");{let mut cache_layers = self.context.cache_layers.borrow_mut();cache_layers.update_after_acceptance(accepted_token_indices,suffix_start,- &command_buffer,+ &mut command_buffer,&self.context.kv_cache_update,);}diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex 1e97a1207..02912efbd 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -34,13 +34,13 @@ use crate::{pub struct AsyncBuffers<B: Backend> {/// Positions buffer: [max_tokens] i32/// Pre-populated with [prefill_count, prefill_count+1, ...]- pub positions: Rc<B::NativeBuffer>,+ pub positions: Rc<RefCell<B::NativeBuffer>>,/// Seeds buffer: [max_tokens] u64/// Pre-populated with deterministic seed sequence- pub seeds: Rc<B::NativeBuffer>,+ pub seeds: Rc<RefCell<B::NativeBuffer>>,/// Results buffer: [batch_size] u32/// Each pass writes its sampled token to results[pass_idx % batch_size]- pub results: Rc<B::NativeBuffer>,+ pub results: Rc<RefCell<B::NativeBuffer>>,/// Event for GPU-side synchronization between passespub event: B::Event,/// Current event counter (pass N waits on N, signals N+1)@@ -66,9 +66,9 @@ impl<B: Backend> AsyncBuffers<B> {let event = context.create_event().expect("Failed to create event");Self {- positions: Rc::new(positions),- seeds: Rc::new(seeds),- results: Rc::new(results),+ positions: Rc::new(RefCell::new(positions)),+ seeds: Rc::new(RefCell::new(seeds)),+ results: Rc::new(RefCell::new(results)),event,counter: Cell::new(0),prefill_count: Cell::new(0),@@ -85,7 +85,7 @@ impl<B: Backend> AsyncBuffers<B> {) {self.prefill_count.set(prefill_count);let base_position = prefill_count.saturating_sub(1);- let ptr = self.positions.cpu_ptr().as_ptr() as *mut i32;+ let ptr = self.positions.borrow().cpu_ptr().as_ptr() as *mut i32;for i in 0..tokens_to_generate {unsafe {*ptr.add(i) = (base_position + i) as i32;@@ -100,7 +100,7 @@ impl<B: Backend> AsyncBuffers<B> {prefix_len: usize,tokens_to_generate: usize,) {- let ptr = self.seeds.cpu_ptr().as_ptr() as *mut u64;+ let ptr = self.seeds.borrow().cpu_ptr().as_ptr() as *mut u64;for i in 0..tokens_to_generate {unsafe {*ptr.add(i) = seed.derive((prefix_len + i - 1) as u64);@@ -118,7 +118,7 @@ impl<B: Backend> AsyncBuffers<B> {&self,pass_idx: usize,) -> u32 {- let ptr = self.results.cpu_ptr().as_ptr() as *const u32;+ let ptr = self.results.borrow().cpu_ptr().as_ptr() as *const u32;unsafe { *ptr.add(pass_idx % self.batch_size) }}}diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex e34b9ac29..fc419fa14 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -1,4 +1,4 @@-use std::{mem::size_of, ops::Deref};+use std::{mem::size_of, ops::DerefMut};use super::LanguageModelGeneratorContext;use crate::{@@ -17,7 +17,7 @@ impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {&self.predicate_buffer}- pub fn disable_execution(&self) {+ pub fn disable_execution(&mut self) {let ptr = self.predicate_buffer.cpu_ptr().as_ptr() as *mut u32;unsafe {@@ -114,7 +114,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {parameters: &EncodingParameters<B>,key: String,) -> LanguageModelGeneratorEncodedTask<B> {- context.executables.encode(state, parameters, context.command_buffer.borrow().deref());+ context.executables.encode(state, parameters, context.command_buffer.borrow_mut().deref_mut());LanguageModelGeneratorEncodedTask {key,diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 18a6d4897..c3e3f5d43 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -282,9 +282,9 @@ whereNone,);- let command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");+ let mut command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");- ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &command_buffer);+ ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &mut command_buffer);command_buffer.submit();command_buffer.wait_until_completed();diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex 488953452..4573536af 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -1,6 +1,7 @@#![cfg(any(target_os = "macos", target_os = "ios"))]use metal::{MTLCommandBuffer, MTLCommandQueue};+use objc2::Message;use uzu::{DataType,array::ArrayContextExt,@@ -190,7 +191,7 @@ fn run_scenario(layer.update_after_acceptance(&scenario.accepted_suffix_indices,scenario.suffix_start,- &root_command_buffer,+ &mut root_command_buffer.retain(),&kv_cache_update,);diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rsindex 6ce00197d..90b93f9d6 100644--- a/crates/uzu/tests/kv_cache_update_test.rs+++ b/crates/uzu/tests/kv_cache_update_test.rs@@ -1,6 +1,6 @@#![cfg(any(target_os = "macos", target_os = "ios"))]-use std::rc::Rc;+use std::{cell::RefCell, rc::Rc};use bytemuck;use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};@@ -81,23 +81,23 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {let device = &context.device;- let key_buffer = Rc::new(+ let key_buffer = Rc::new(RefCell::new(device.new_buffer_with_data(bytemuck::cast_slice(key_data.as_slice().unwrap()),MTLResourceOptions::STORAGE_MODE_SHARED,).expect("Failed to create buffer"),- );+ ));- let value_buffer = Rc::new(+ let value_buffer = Rc::new(RefCell::new(device.new_buffer_with_data(bytemuck::cast_slice(value_data.as_slice().unwrap()),MTLResourceOptions::STORAGE_MODE_SHARED,).expect("Failed to create buffer"),- );+ ));let kv_layer_data = KVLayerData::<Metal> {key_buffer: key_buffer.clone(),@@ -106,8 +106,9 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {value_shape: [num_heads, seq_len, head_dim],};- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &command_buffer) {+ let mut command_buffer =+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();+ match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut command_buffer) {Ok(_) => {},Err(e) => {println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);@@ -118,8 +119,8 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {command_buffer.commit();command_buffer.wait_until_completed();- let key_result_ptr = key_buffer.contents().as_ptr() as *const f32;- let value_result_ptr = value_buffer.contents().as_ptr() as *const f32;+ let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;+ let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;let total_elems = num_heads * seq_len * head_dim;diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rsindex b6a53be53..874a4b28b 100644--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs@@ -346,7 +346,7 @@ fn run_moe_parity_test_internal(// Encode ALL kernels in one command buffereprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");// Router + TopK (fused kernel)let router_topk =@@ -412,7 +412,7 @@ fn run_moe_parity_test_internal(let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");gather.encode(- &cb,+ &mut cb,DataType::BF16,&MoeGatherArguments {x_buffer: &x_buf,@@ -427,16 +427,16 @@ fn run_moe_parity_test_internal(// Additional buffers for 2-passlet total_rows = t * k;- let hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);let row_expert_map_buf = alloc_buffer::<u32>(&ctx, total_rows);let experts = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("experts");let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;- let args = MoeExpertsTwoPassArguments {+ let mut args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,+ hidden_buffer: &mut hidden_buf,output_buffer: &y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,@@ -460,7 +460,7 @@ fn run_moe_parity_test_internal(silu_alpha,data_type: DataType::BF16,};- experts.encode(&cb, &args);+ experts.encode(&mut cb, &mut args);let finalize =<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rsindex 07f20f91b..a71375bc2 100644--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs@@ -93,7 +93,7 @@ fn run_decode_case(let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);// Buffers for indirect dispatch@@ -105,38 +105,39 @@ fn run_decode_case(let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);- let make_two_pass_args = || MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &output_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- data_type: DataType::BF16,- };-for _ in 0..warmup {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, &make_two_pass_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ experts_kernel.encode(+ &mut cb,+ &MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &mut hidden_buf,+ output_buffer: &output_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.702,+ data_type: DataType::BF16,+ },+ );cb.commit();cb.wait_until_completed();}@@ -144,8 +145,38 @@ fn run_decode_case(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, &make_two_pass_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ experts_kernel.encode(+ &mut cb,+ &MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &mut hidden_buf,+ output_buffer: &output_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.702,+ data_type: DataType::BF16,+ },+ );cb.commit();cb.wait_until_completed();times.push(start.elapsed().as_secs_f64() * 1000.0);@@ -227,7 +258,7 @@ fn run_two_pass_prefill_case(let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);@@ -238,38 +269,37 @@ fn run_two_pass_prefill_case(let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);- let make_two_pass_args = || MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &output_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.702,- data_type: DataType::BF16,- };-for _ in 0..warmup {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, &make_two_pass_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut args = MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &mut hidden_buf,+ output_buffer: &output_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.702,+ data_type: DataType::BF16,+ };+ experts_kernel.encode(&mut cb, &mut args);cb.commit();cb.wait_until_completed();}@@ -277,8 +307,36 @@ fn run_two_pass_prefill_case(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, &make_two_pass_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut args = MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &mut hidden_buf,+ output_buffer: &output_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.702,+ data_type: DataType::BF16,+ };+ experts_kernel.encode(&mut cb, &mut args);cb.commit();cb.wait_until_completed();times.push(start.elapsed().as_secs_f64() * 1000.0);@@ -377,8 +435,8 @@ fn run_fused_single_token_case(};for _ in 0..warmup {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- fused_kernel.encode(&cb, make_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ fused_kernel.encode(&mut cb, make_args());cb.commit();cb.wait_until_completed();}@@ -386,8 +444,8 @@ fn run_fused_single_token_case(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- fused_kernel.encode(&cb, make_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ fused_kernel.encode(&mut cb, make_args());cb.commit();cb.wait_until_completed();times.push(start.elapsed().as_secs_f64() * 1000.0);@@ -499,7 +557,7 @@ fn run_indirect_decode_timed(let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;- let hidden_buf = alloc_buffer::<f32>(ctx, sum_k * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(ctx, sum_k * d_ff);let output_buf = alloc_buffer::<bf16>(ctx, sum_k * d_model);let tile_counts_buf = alloc_buffer::<u32>(ctx, e);@@ -510,38 +568,39 @@ fn run_indirect_decode_timed(let dispatch_args_buf = alloc_buffer::<u32>(ctx, 3);let row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);- let make_args = || MoeExpertsTwoPassArguments {- x_perm_buffer: &x_perm_buf,- expert_offsets: &offsets_buf,- row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,- output_buffer: &output_buf,- w13_all: &w13_buf,- w2_all: &w2_buf,- up_biases: &up_biases_buf,- down_biases: &down_biases_buf,- tile_counts: &tile_counts_buf,- tile_offsets: &tile_offsets_buf,- tile_map: &tile_map_buf,- total_tiles: &total_tiles_buf,- dispatch_args: &dispatch_args_buf,- total_rows: sum_k,- d_model,- d_ff,- e,- num_tiles_k: num_tiles_k as u32,- gating_code: 2,- gate_clip_min: f32::NEG_INFINITY,- gate_clip_max: f32::INFINITY,- up_clip_min: f32::NEG_INFINITY,- up_clip_max: f32::INFINITY,- silu_alpha: 1.0,- data_type: DataType::BF16,- };-for _ in 0..warmup {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, &make_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ experts_kernel.encode(+ &mut cb,+ &MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &mut hidden_buf,+ output_buffer: &output_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.0,+ data_type: DataType::BF16,+ },+ );cb.commit();cb.wait_until_completed();}@@ -549,8 +608,38 @@ fn run_indirect_decode_timed(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- experts_kernel.encode(&cb, &make_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ experts_kernel.encode(+ &mut cb,+ &MoeExpertsTwoPassArguments {+ x_perm_buffer: &x_perm_buf,+ expert_offsets: &offsets_buf,+ row_expert_map: &row_expert_map_buf,+ hidden_buffer: &mut hidden_buf,+ output_buffer: &output_buf,+ w13_all: &w13_buf,+ w2_all: &w2_buf,+ up_biases: &up_biases_buf,+ down_biases: &down_biases_buf,+ tile_counts: &tile_counts_buf,+ tile_offsets: &tile_offsets_buf,+ tile_map: &tile_map_buf,+ total_tiles: &total_tiles_buf,+ dispatch_args: &dispatch_args_buf,+ total_rows: sum_k,+ d_model,+ d_ff,+ e,+ num_tiles_k: num_tiles_k as u32,+ gating_code: 2,+ gate_clip_min: f32::NEG_INFINITY,+ gate_clip_max: f32::INFINITY,+ up_clip_min: f32::NEG_INFINITY,+ up_clip_max: f32::INFINITY,+ silu_alpha: 1.0,+ data_type: DataType::BF16,+ },+ );cb.commit();cb.wait_until_completed();times.push(start.elapsed().as_secs_f64() * 1000.0);@@ -621,8 +710,8 @@ fn run_fused_decode_timed(};for _ in 0..warmup {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- fused_kernel.encode(&cb, make_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ fused_kernel.encode(&mut cb, make_args());cb.commit();cb.wait_until_completed();}@@ -630,8 +719,8 @@ fn run_fused_decode_timed(let mut times = Vec::with_capacity(iters);for _ in 0..iters {let start = Instant::now();- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- fused_kernel.encode(&cb, make_args());+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ fused_kernel.encode(&mut cb, make_args());cb.commit();cb.wait_until_completed();times.push(start.elapsed().as_secs_f64() * 1000.0);diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rsindex 8bcd8bc53..3222ef730 100644--- a/crates/uzu/tests/moe/moe_experts_test.rs+++ b/crates/uzu/tests/moe/moe_experts_test.rs@@ -401,7 +401,7 @@ fn test_two_pass_decode_correctness() {let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);// Tile infrastructure@@ -416,18 +416,18 @@ fn test_two_pass_decode_correctness() {// Execute 2-pass decode kernellet experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");const K_TILE: usize = 64;let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;experts_kernel.encode(- &cb,+ &mut cb,&MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,+ hidden_buffer: &mut hidden_buf,output_buffer: &y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,@@ -548,7 +548,7 @@ fn test_two_pass_decode_multi_token() {let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);// Tile infrastructure@@ -560,14 +560,14 @@ fn test_two_pass_decode_multi_token() {let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(- &cb,+ &mut cb,&MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,+ hidden_buffer: &mut hidden_buf,output_buffer: &y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,@@ -650,7 +650,7 @@ fn test_two_pass_prefill_correctness() {let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);// Tile infrastructure@@ -662,12 +662,12 @@ fn test_two_pass_prefill_correctness() {let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");- let args = MoeExpertsTwoPassArguments {+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut args = MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,+ hidden_buffer: &mut hidden_buf,output_buffer: &y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,@@ -691,7 +691,7 @@ fn test_two_pass_prefill_correctness() {silu_alpha,data_type: DataType::BF16,};- experts_kernel.encode(&cb, &args);+ experts_kernel.encode(&mut cb, &mut args);cb.commit();cb.wait_until_completed();@@ -791,10 +791,10 @@ fn test_fused_single_token_decode() {// Run fused decode kernellet fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(- &cb,+ &mut cb,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,@@ -915,9 +915,9 @@ fn test_fused_single_token_k4() {let y_buf = alloc_buffer::<bf16>(&ctx, d_model);let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");fused_kernel.encode(- &cb,+ &mut cb,MoeExpertsSingleDecodeArguments {x: &x_buf,topk_ids: &topk_ids_buf,diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rsindex 928823bd9..62ef950a5 100644--- a/crates/uzu/tests/moe/moe_gather_test.rs+++ b/crates/uzu/tests/moe/moe_gather_test.rs@@ -78,9 +78,9 @@ fn test_gather_correctness() {// Execute gather kernel using kernel structlet gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");gather.encode(- &cb,+ &mut cb,DataType::BF16,&MoeGatherArguments {x_buffer: &x_buf,diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rsindex a81c6af36..f62468bb1 100644--- a/crates/uzu/tests/moe/moe_perf_test.rs+++ b/crates/uzu/tests/moe/moe_perf_test.rs@@ -285,7 +285,7 @@ fn test_moe_pipeline_breakdown_decode() {let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);// Two-pass specific buffers- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);let row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);// Scatter block bases buffers@@ -388,9 +388,9 @@ fn test_moe_pipeline_breakdown_decode() {});let gather_perf = time_kernel("Gather", 2, 5, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");gather_kernel.encode(- &cb,+ &mut cb,DataType::BF16,&MoeGatherArguments {x_buffer: &x_buf,@@ -407,14 +407,14 @@ fn test_moe_pipeline_breakdown_decode() {});let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");experts_kernel.encode(- &cb,+ &mut cb,&MoeExpertsTwoPassArguments {x_perm_buffer: &x_perm_buf,expert_offsets: &offsets_buf,row_expert_map: &row_expert_map_buf,- hidden_buffer: &hidden_buf,+ hidden_buffer: &mut hidden_buf,output_buffer: &y_partial_buf,w13_all: &w13_buf,w2_all: &w2_buf,diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rsindex 874d41329..d72ddfd02 100644--- a/crates/uzu/tests/moe/moe_tiles_test.rs+++ b/crates/uzu/tests/moe/moe_tiles_test.rs@@ -35,9 +35,9 @@ fn test_tile_counts_correctness() {// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(- &cb,+ &mut cb,&MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &tile_counts_buf,@@ -80,9 +80,9 @@ fn test_tile_scan_correctness() {// Execute kernel using kernel structlet tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_scan(- &cb,+ &mut cb,&MoeTileScanArguments {tile_counts_buffer: &tile_counts_buf,tile_offsets_buffer: &tile_offsets_buf,@@ -120,9 +120,9 @@ fn test_tile_edge_cases() {let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(- &cb,+ &mut cb,&MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &tile_counts_buf,@@ -151,9 +151,9 @@ fn test_tile_edge_cases() {let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");tile_kernel.encode_counts(- &cb,+ &mut cb,&MoeTileCountsArguments {offsets_buffer: &offsets_buf,tile_counts_buffer: &tile_counts_buf,diff --git a/crates/uzu/tests/person_schema_metal_bitmask_test.rs b/crates/uzu/tests/person_schema_metal_bitmask_test.rsindex 1b181ffe7..4e4e1fcc6 100644--- a/crates/uzu/tests/person_schema_metal_bitmask_test.rs+++ b/crates/uzu/tests/person_schema_metal_bitmask_test.rs@@ -1,6 +1,6 @@#![cfg(target_os = "macos")]-use std::{path::PathBuf, rc::Rc};+use std::{cell::RefCell, path::PathBuf, rc::Rc};use metal::{MTLDevice, MTLDeviceExt, MTLResourceOptions};use schemars::JsonSchema;@@ -62,7 +62,7 @@ fn person_schema_metal_bitmask() {let elems = batch * buffer_size;let bytes = elems * core::mem::size_of::<i32>();let buffer = device.new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let metal_bitmask = unsafe { Array::<Metal>::from_parts(Rc::new(buffer), 0, &shape, DataType::I32) };+ let metal_bitmask = unsafe { Array::<Metal>::from_parts(Rc::new(RefCell::new(buffer)), 0, &shape, DataType::I32) };let mut shape_i64 = [buffer_size as i64];let mut bitmask_tensor = DLTensor {