Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Backend structs mutability (#228)

Aleksandr Golokozc00ed108236d
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index b7f677d03..76060401e 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -320,13 +320,13 @@ pub fn bindgen(
Ok(Self { pipeline #(, #conditional_buffer_sets)* })
}
- fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder ProtocolObject<dyn MTLComputeCommandEncoder>) {
+ fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>) {
#empty_dispatch_guards
compute_encoder.set_compute_pipeline_state(&self.pipeline);
#(#encode_args_sets)*
#dispatch
}
- fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder ProtocolObject<dyn MTLComputeCommandEncoder>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {
+ fn encode_if<#(#encode_generics, )* 'encoder, 'predicate>(&self, #(#encode_args_defs, )* compute_encoder: &'encoder Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>, predicate: Option<impl crate::backends::common::kernel::BufferArg<'predicate, Retained<ProtocolObject<dyn MTLBuffer>>>>) {
#empty_dispatch_guards
if let Some(predicate) = predicate {
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rs
index 730b54686..9b9b658d8 100644
--- a/crates/uzu/src/array.rs
+++ b/crates/uzu/src/array.rs
@@ -9,7 +9,7 @@ use crate::{
#[derive(Debug)]
pub struct Array<B: Backend> {
- buffer: Rc<B::NativeBuffer>,
+ buffer: Rc<RefCell<B::NativeBuffer>>,
offset: usize,
shape: Box<[usize]>,
data_type: DataType,
@@ -18,20 +18,20 @@ pub struct Array<B: Backend> {
impl<B: Backend> Array<B> {
// Constructors
pub unsafe fn from_parts(
- buffer: Rc<B::NativeBuffer>,
+ buffer: Rc<RefCell<B::NativeBuffer>>,
offset: usize,
shape: &[usize],
data_type: DataType,
) -> Self {
let required_bytes = size_for_shape(shape, data_type);
assert!(
- offset + required_bytes <= buffer.length(),
+ offset + required_bytes <= buffer.borrow().length(),
"Shape {:?} with data type {:?} at offset {} requires {} bytes total, but buffer length is {} bytes",
shape,
data_type,
offset,
offset + required_bytes,
- buffer.length()
+ buffer.borrow().length()
);
Self {
buffer: buffer.clone(),
@@ -49,11 +49,7 @@ impl<B: Backend> Array<B> {
}
// Getters
- pub fn buffer(&self) -> &B::NativeBuffer {
- &self.buffer.as_ref()
- }
-
- pub fn buffer_rc(&self) -> Rc<B::NativeBuffer> {
+ pub fn buffer(&self) -> Rc<RefCell<B::NativeBuffer>> {
self.buffer.clone()
}
@@ -71,7 +67,7 @@ impl<B: Backend> Array<B> {
// Utility
pub fn cpu_ptr(&self) -> NonNull<c_void> {
- unsafe { self.buffer.cpu_ptr().add(self.offset) }
+ unsafe { self.buffer.borrow().cpu_ptr().add(self.offset) }
}
pub fn size(&self) -> usize {
@@ -278,6 +274,6 @@ impl<C: Context> ArrayContextExt for C {
let mut buffer = self.create_buffer(buffer_size_bytes).expect("Failed to create buffer");
buffer.set_label(Some(label));
- unsafe { Array::from_parts(Rc::new(buffer), 0, shape, data_type) }
+ unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, shape, data_type) }
}
}
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index 363d77d86..fed9629f1 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -4,13 +4,13 @@ pub trait CommandBuffer {
type Backend: Backend;
fn with_compute_encoder<T>(
- &self,
- callback: impl FnOnce(&<Self::Backend as Backend>::ComputeEncoder) -> T,
+ &mut self,
+ callback: impl FnOnce(&mut <Self::Backend as Backend>::ComputeEncoder) -> T,
) -> T;
fn with_copy_encoder<T>(
- &self,
- callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,
+ &mut self,
+ callback: impl FnOnce(&mut <Self::Backend as Backend>::CopyEncoder) -> T,
) -> T;
fn encode_wait_for_event(
diff --git a/crates/uzu/src/backends/common/copy_encoder.rs b/crates/uzu/src/backends/common/copy_encoder.rs
index 9b39ac447..93c319ae2 100644
--- a/crates/uzu/src/backends/common/copy_encoder.rs
+++ b/crates/uzu/src/backends/common/copy_encoder.rs
@@ -6,15 +6,15 @@ pub trait CopyEncoder {
type Backend: Backend;
fn encode_copy(
- &self,
+ &mut self,
src: &<Self::Backend as Backend>::NativeBuffer,
- dst: &<Self::Backend as Backend>::NativeBuffer,
+ dst: &mut <Self::Backend as Backend>::NativeBuffer,
size: usize,
);
fn encode_fill(
- &self,
- dst: &<Self::Backend as Backend>::NativeBuffer,
+ &mut self,
+ dst: &mut <Self::Backend as Backend>::NativeBuffer,
range: Range<usize>,
value: u8,
);
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
index bb89ea6e8..839b637ea 100644
--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -1,4 +1,4 @@
-use std::{mem::size_of, rc::Rc};
+use std::{cell::RefCell, mem::size_of, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -8,9 +8,9 @@ use crate::{
};
pub struct KVLayerData<B: Backend> {
- pub key_buffer: Rc<B::NativeBuffer>,
+ pub key_buffer: Rc<RefCell<B::NativeBuffer>>,
pub key_shape: [usize; 3],
- pub value_buffer: Rc<B::NativeBuffer>,
+ pub value_buffer: Rc<RefCell<B::NativeBuffer>>,
pub value_shape: [usize; 3],
}
@@ -57,7 +57,7 @@ impl<B: Backend> KVCacheUpdate<B> {
in_place_data: &[KVLayerData<B>],
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) -> Result<(), KVCacheUpdateError<B>> {
command_buffer.with_compute_encoder(|encoder| {
self.encode_with_encoder(in_place_data, source_indices, destination_indices, encoder)
@@ -92,8 +92,8 @@ impl<B: Backend> KVCacheUpdate<B> {
// non-inline is not supported yet (and is broken anyways due to a data race)
for swaps_chunk in swaps.chunks(max_inline_swaps) {
self.kernel.encode(
- layer_data.key_buffer.as_ref(),
- layer_data.value_buffer.as_ref(),
+ layer_data.key_buffer.borrow().deref(),
+ layer_data.value_buffer.borrow().deref(),
swaps_chunk,
swaps_chunk.len() as u32,
num_heads as u32,
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
index cc408d358..70ca68396 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_single.rs
@@ -82,7 +82,7 @@ impl<B: Backend> MoeExpertsSingleDecodeKernels<B> {
pub fn encode(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: MoeExpertsSingleDecodeArguments<B>,
) {
if args.k == 0 {
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
index 9650dbe5d..a366d82cd 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_decode.rs
@@ -1,3 +1,5 @@
+use std::ops::Deref;
+
use super::{
MoeExpertsTwoPassArguments, MoePassARowMapArguments, MoePassATileBuildArguments, MoePassATileCountsArguments,
MoePassATileDispatchArguments, MoePassATileKernels, MoePassATileScanArguments,
@@ -45,7 +47,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
pub fn encode(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoeExpertsTwoPassArguments<B>,
) {
if args.total_rows == 0 {
@@ -112,7 +114,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
args.x_perm_buffer,
args.expert_offsets,
args.w13_all,
- args.hidden_buffer,
+ args.hidden_buffer.deref(),
args.up_biases,
args.d_model as u32,
args.d_ff as u32,
@@ -132,7 +134,7 @@ impl<B: Backend> MoeExpertsTwoPassDecodeBlock<B> {
command_buffer.with_compute_encoder(|encoder| {
let pass_b_kernel = &self.fused_down[dtype_idx];
pass_b_kernel.encode(
- args.hidden_buffer,
+ args.hidden_buffer.deref(),
args.row_expert_map,
args.w2_all,
args.down_biases,
diff --git a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
index c82d14345..4d0584565 100644
--- a/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/experts_two_pass_prefill.rs
@@ -1,3 +1,5 @@
+use std::ops::Deref;
+
use super::{
MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels, MoeTileScanArguments,
};
@@ -46,8 +48,8 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
pub fn encode(
&self,
- command_buffer: &B::CommandBuffer,
- args: &MoeExpertsTwoPassArguments<B>,
+ command_buffer: &mut B::CommandBuffer,
+ args: &mut MoeExpertsTwoPassArguments<B>,
) {
if args.total_rows == 0 {
return;
@@ -116,7 +118,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
args.expert_offsets,
args.w13_all,
args.up_biases,
- args.hidden_buffer,
+ args.hidden_buffer.deref(),
args.d_model as u32,
args.d_ff as u32,
args.e as u32,
@@ -141,7 +143,7 @@ impl<B: Backend> MoeExpertsTwoPassPrefillBlock<B> {
command_buffer.with_compute_encoder(|encoder| {
let kernel_pass_b = &self.pass_b_indirect[dtype_idx];
kernel_pass_b.encode(
- args.hidden_buffer,
+ args.hidden_buffer.deref(),
args.expert_offsets,
args.w2_all,
args.down_biases,
@@ -162,7 +164,7 @@ pub struct MoeExpertsTwoPassArguments<'a, B: Backend> {
pub x_perm_buffer: &'a B::NativeBuffer,
pub expert_offsets: &'a B::NativeBuffer,
pub row_expert_map: &'a B::NativeBuffer,
- pub hidden_buffer: &'a B::NativeBuffer,
+ pub hidden_buffer: &'a mut B::NativeBuffer,
pub output_buffer: &'a B::NativeBuffer,
pub w13_all: &'a B::NativeBuffer,
pub w2_all: &'a B::NativeBuffer,
diff --git a/crates/uzu/src/backends/common/kernel/moe/gather.rs b/crates/uzu/src/backends/common/kernel/moe/gather.rs
index bd4bc07bb..b812598d0 100644
--- a/crates/uzu/src/backends/common/kernel/moe/gather.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/gather.rs
@@ -34,7 +34,7 @@ impl<B: Backend> MoeGatherKernels<B> {
pub fn encode(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
dtype: DataType,
args: &MoeGatherArguments<B>,
) {
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
index 3590cb360..8a4c90bd9 100644
--- a/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_map.rs
@@ -22,7 +22,7 @@ impl<B: Backend> MoeTileMapKernels<B> {
pub fn encode_counts(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoeTileCountsArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
@@ -32,7 +32,7 @@ impl<B: Backend> MoeTileMapKernels<B> {
pub fn encode_scan(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoeTileScanArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
@@ -48,7 +48,7 @@ impl<B: Backend> MoeTileMapKernels<B> {
pub fn encode_build_map(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoeTileMapBuildArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
@@ -65,7 +65,7 @@ impl<B: Backend> MoeTileMapKernels<B> {
pub fn encode_dispatch_args(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoeTileDispatchArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
diff --git a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
index 43548c713..106a9f657 100644
--- a/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
+++ b/crates/uzu/src/backends/common/kernel/moe/tiles_pass_a.rs
@@ -68,7 +68,7 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_counts(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoePassATileCountsArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
@@ -78,7 +78,7 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_scan(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoePassATileScanArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
@@ -88,7 +88,7 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_row_map(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoePassARowMapArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
@@ -104,7 +104,7 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_build_map(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoePassATileBuildArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
@@ -122,7 +122,7 @@ impl<B: Backend> MoePassATileKernels<B> {
pub fn encode_dispatch_args(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
args: &MoePassATileDispatchArguments<B>,
) {
command_buffer.with_compute_encoder(|encoder| {
diff --git a/crates/uzu/src/backends/metal/backend.rs b/crates/uzu/src/backends/metal/backend.rs
index fc5d832ee..c8e37bb01 100644
--- a/crates/uzu/src/backends/metal/backend.rs
+++ b/crates/uzu/src/backends/metal/backend.rs
@@ -11,8 +11,8 @@ impl Backend for Metal {
type Context = MetalContext;
type NativeBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;
type CommandBuffer = Retained<ProtocolObject<dyn MTLCommandBuffer>>;
- type ComputeEncoder = ProtocolObject<dyn MTLComputeCommandEncoder>;
- type CopyEncoder = ProtocolObject<dyn MTLBlitCommandEncoder>;
+ type ComputeEncoder = Retained<ProtocolObject<dyn MTLComputeCommandEncoder>>;
+ type CopyEncoder = Retained<ProtocolObject<dyn MTLBlitCommandEncoder>>;
type Event = Retained<ProtocolObject<dyn MTLEvent>>;
type Kernels = MetalKernels;
type Error = MetalError;
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index ab89ccd33..278300aa9 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -10,12 +10,12 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
type Backend = Metal;
fn with_compute_encoder<T>(
- &self,
- callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,
+ &mut self,
+ callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::ComputeEncoder) -> T,
) -> T {
- let encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");
+ let mut encoder = self.new_compute_command_encoder().expect("Failed to create compute command encoder");
- let ret = callback(&encoder);
+ let ret = callback(&mut encoder);
encoder.end_encoding();
@@ -23,12 +23,12 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
}
fn with_copy_encoder<T>(
- &self,
- callback: impl FnOnce(&<Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,
+ &mut self,
+ callback: impl FnOnce(&mut <Self::Backend as crate::backends::common::Backend>::CopyEncoder) -> T,
) -> T {
- let encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");
+ let mut encoder = self.new_blit_command_encoder().expect("Failed to create blit command encoder");
- let ret = callback(&encoder);
+ let ret = callback(&mut encoder);
encoder.end_encoding();
diff --git a/crates/uzu/src/backends/metal/copy_encoder.rs b/crates/uzu/src/backends/metal/copy_encoder.rs
index 12cf03bff..e74776e15 100644
--- a/crates/uzu/src/backends/metal/copy_encoder.rs
+++ b/crates/uzu/src/backends/metal/copy_encoder.rs
@@ -1,18 +1,18 @@
use std::ops::Range;
use metal::{MTLBlitCommandEncoder, MTLBlitCommandEncoderExt, MTLBuffer};
-use objc2::runtime::ProtocolObject;
+use objc2::{rc::Retained, runtime::ProtocolObject};
use super::Metal;
use crate::backends::common::{Backend, CopyEncoder};
-impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {
+impl CopyEncoder for Retained<ProtocolObject<dyn MTLBlitCommandEncoder>> {
type Backend = Metal;
fn encode_copy(
- &self,
+ &mut self,
src: &<Self::Backend as Backend>::NativeBuffer,
- dst: &<Self::Backend as Backend>::NativeBuffer,
+ dst: &mut <Self::Backend as Backend>::NativeBuffer,
size: usize,
) {
assert!(src.length() >= size && dst.length() >= size);
@@ -21,8 +21,8 @@ impl CopyEncoder for ProtocolObject<dyn MTLBlitCommandEncoder> {
}
fn encode_fill(
- &self,
- dst: &<Self::Backend as Backend>::NativeBuffer,
+ &mut self,
+ dst: &mut <Self::Backend as Backend>::NativeBuffer,
range: Range<usize>,
value: u8,
) {
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 941c994b3..479d205f7 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -111,35 +111,35 @@ where
let encoding_params = EncodingParameters::new(false, true, false);
- self.context.embed.encode(&mut state, &encoding_params, &self.context.command_buffer);
- self.context.embedding_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ self.context.embed.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
+ self.context.embedding_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
state.encode_copy_array(
- &self.context.command_buffer,
+ &mut self.context.command_buffer,
ArrayId::Main,
traces.borrow().embedding_norm().clone(),
);
}
for layer in self.context.layers.iter() {
- layer.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ layer.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
}
- self.context.output_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ self.context.output_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
state.encode_copy_array(
- &self.context.command_buffer,
+ &mut self.context.command_buffer,
ArrayId::Main,
traces.borrow().output_norm.clone(),
);
}
- self.context.pooling.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ self.context.pooling.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
- self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ self.context.prediction_head.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
self.context.command_buffer.submit();
self.context.command_buffer.wait_until_completed();
@@ -174,14 +174,14 @@ where
let encoding_params = EncodingParameters::new(false, true, false);
- self.context.embed.encode(&mut state, &encoding_params, &self.context.command_buffer);
- self.context.embedding_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ self.context.embed.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
+ self.context.embedding_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
for layer in self.context.layers.iter() {
- layer.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ layer.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
}
- self.context.output_norm.encode(&mut state, &encoding_params, &self.context.command_buffer);
- self.context.pooling.encode(&mut state, &encoding_params, &self.context.command_buffer);
- self.context.prediction_head.encode(&mut state, &encoding_params, &self.context.command_buffer);
+ self.context.output_norm.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
+ self.context.pooling.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
+ self.context.prediction_head.encode(&mut state, &encoding_params, &mut self.context.command_buffer);
self.context.command_buffer.submit();
self.context.command_buffer.wait_until_completed();
diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rs
index dd09a8315..133866eaa 100644
--- a/crates/uzu/src/encodable_block/activation.rs
+++ b/crates/uzu/src/encodable_block/activation.rs
@@ -1,5 +1,7 @@
//! Activation encodable.
+use std::ops::Deref;
+
use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
@@ -60,6 +62,12 @@ impl<B: Backend> EncodableBlock<B> for Activation<B> {
ActivationConfig::Identity => panic!("Identity activation is not supported for kernel"),
};
- self.kernel.encode(input_array.buffer(), output_array.buffer(), n as u32, act_type, encoder);
+ self.kernel.encode(
+ input_array.buffer().borrow().deref(),
+ output_array.buffer().borrow().deref(),
+ n as u32,
+ act_type,
+ encoder,
+ );
}
}
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index 39859aceb..444127f32 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -1,6 +1,6 @@
//! Attention kernel encodable.
-use std::collections::HashMap;
+use std::{collections::HashMap, ops::Deref};
use crate::{
DataType,
@@ -132,7 +132,7 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, &encoder));
@@ -221,32 +221,39 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
};
let rotated_keys_array = rotated_keys_binding[0].borrow_mut();
- let rotated_keys_buffer = rotated_keys_array.buffer();
+ let rotated_keys_buf_rc = rotated_keys_array.buffer();
+ let rotated_keys_buf_borrow = rotated_keys_buf_rc.borrow();
let qkv_array = qkv_binding[0].borrow_mut();
- let qkv_buffer = qkv_array.buffer();
+ let qkv_buf_rc = qkv_array.buffer();
+ let qkv_buf_borrow = qkv_buf_rc.borrow();
// Get KV cache buffers only if KV cache exists (LLM mode)
let has_kv_cache = state.cache_layers().is_some();
let key_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Keys(self.layer_index)]));
let key_cache_array = key_cache_binding.as_ref().map(|b| b[0].borrow_mut());
+ let key_cache_buf_rc = key_cache_array.as_ref().map(|a| a.buffer());
+ let key_cache_buf_borrow = key_cache_buf_rc.as_ref().map(|rc| rc.borrow());
let value_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Values(self.layer_index)]));
let value_cache_array = value_cache_binding.as_ref().map(|b| b[0].borrow_mut());
+ let value_cache_buf_rc = value_cache_array.as_ref().map(|a| a.buffer());
+ let value_cache_buf_borrow = value_cache_buf_rc.as_ref().map(|rc| rc.borrow());
let extracted_values_binding = (!has_kv_cache).then(|| state.arrays(&[ArrayId::ExtractedValues]));
let extracted_values_array = extracted_values_binding.as_ref().map(|b| b[0].borrow_mut());
+ let extracted_values_buf_rc = extracted_values_array.as_ref().map(|a| a.buffer());
+ let extracted_values_buf_borrow = extracted_values_buf_rc.as_ref().map(|rc| rc.borrow());
// For classifiers (no KV cache): extract values from QKV into a dedicated extracted_values buffer.
if !has_kv_cache {
- let extracted_values_buf = extracted_values_array.as_ref().unwrap().buffer();
self.update_kv_cache_kernel.encode(
- rotated_keys_buffer,
- qkv_buffer,
+ rotated_keys_buf_borrow.deref(),
+ qkv_buf_borrow.deref(),
// keys already in desired layout; harmless overwrite
- rotated_keys_buffer,
- extracted_values_buf,
+ rotated_keys_buf_borrow.deref(),
+ extracted_values_buf_borrow.as_ref().unwrap().deref(),
num_groups as u32,
num_heads as u32,
head_dim as u32,
@@ -257,22 +264,24 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
);
}
- let key_cache_buffer = if has_kv_cache {
- key_cache_array.as_ref().unwrap().buffer()
+ let key_cache_buffer: &B::NativeBuffer = if has_kv_cache {
+ key_cache_buf_borrow.as_ref().unwrap().deref()
} else {
- rotated_keys_buffer
+ rotated_keys_buf_borrow.deref()
};
- let value_cache_buffer = if has_kv_cache {
- value_cache_array.as_ref().unwrap().buffer()
+ let value_cache_buffer: &B::NativeBuffer = if has_kv_cache {
+ value_cache_buf_borrow.as_ref().unwrap().deref()
} else {
- extracted_values_array.as_ref().unwrap().buffer()
+ extracted_values_buf_borrow.as_ref().unwrap().deref()
};
let queries_array = rotated_queries_binding[0].borrow_mut();
- let queries_buffer = queries_array.buffer();
+ let queries_buf_rc = queries_array.buffer();
+ let queries_buf_borrow = queries_buf_rc.borrow();
let attention_output_array = attention_output_binding[0].borrow_mut();
- let attention_output_buffer = attention_output_array.buffer();
+ let attention_output_buf_rc = attention_output_array.buffer();
+ let attention_output_buf_borrow = attention_output_buf_rc.borrow();
let attention_bias_array_borrow = if use_mask {
Some(
@@ -284,29 +293,36 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
} else {
None
};
- let attention_bias_buffer = attention_bias_array_borrow.as_ref().map(|a| a.buffer());
+ let attention_bias_buf_rc = attention_bias_array_borrow.as_ref().map(|a| a.buffer());
+ let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());
+ let attention_bias_buffer: Option<&B::NativeBuffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());
let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);
let sums_binding = state.arrays(&[ArrayId::AttentionSums]);
let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);
let partials_array = partials_binding[0].borrow_mut();
- let partials_buffer = partials_array.buffer();
+ let partials_buf_rc = partials_array.buffer();
+ let partials_buf_borrow = partials_buf_rc.borrow();
let sums_array = sums_binding[0].borrow_mut();
- let sums_buffer = sums_array.buffer();
+ let sums_buf_rc = sums_array.buffer();
+ let sums_buf_borrow = sums_buf_rc.borrow();
let maxs_array = maxs_binding[0].borrow_mut();
- let maxs_buffer = maxs_array.buffer();
+ let maxs_buf_rc = maxs_array.buffer();
+ let maxs_buf_borrow = maxs_buf_rc.borrow();
let sinks_borrow = sinks_binding.as_ref().map(|binding| binding[0].borrow());
- let sinks_buffer = sinks_borrow.as_ref().map(|b| b.buffer());
+ let sinks_buf_rc = sinks_borrow.as_ref().map(|b| b.buffer());
+ let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());
+ let sinks_buffer: Option<&B::NativeBuffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());
// Only update KV cache for LLM mode (not for classifiers)
if has_kv_cache {
self.update_kv_cache_kernel.encode(
- rotated_keys_buffer,
- qkv_buffer,
+ rotated_keys_buf_borrow.deref(),
+ qkv_buf_borrow.deref(),
key_cache_buffer,
value_cache_buffer,
num_groups as u32,
@@ -341,10 +357,10 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
match variant {
KernelVariant::Gemm => {
let args = AttentionGemmArguments {
- queries_buffer,
+ queries_buffer: queries_buf_borrow.deref(),
keys_buffer: key_cache_buffer,
values_buffer: value_cache_buffer,
- output_buffer: attention_output_buffer,
+ output_buffer: attention_output_buf_borrow.deref(),
mask_buffer: attention_bias_buffer,
sinks_buffer,
num_heads,
@@ -367,10 +383,10 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),
};
kernel.encode(
- queries_buffer,
+ queries_buf_borrow.deref(),
key_cache_buffer,
value_cache_buffer,
- attention_output_buffer,
+ attention_output_buf_borrow.deref(),
gqa_factor as u32,
sequence_length as u32,
k_head_stride as u32,
@@ -398,12 +414,12 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),
};
kernel_pass1.encode(
- queries_buffer,
+ queries_buf_borrow.deref(),
key_cache_buffer,
value_cache_buffer,
- partials_buffer,
- sums_buffer,
- maxs_buffer,
+ partials_buf_borrow.deref(),
+ sums_buf_borrow.deref(),
+ maxs_buf_borrow.deref(),
gqa_factor as u32,
sequence_length as u32,
k_head_stride as u32,
@@ -421,10 +437,10 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
&compute_encoder,
);
kernel_pass2.encode(
- partials_buffer,
- sums_buffer,
- maxs_buffer,
- attention_output_buffer,
+ partials_buf_borrow.deref(),
+ sums_buf_borrow.deref(),
+ maxs_buf_borrow.deref(),
+ attention_output_buf_borrow.deref(),
num_heads as u32,
suffix_length as u32,
&compute_encoder,
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index 9a5d7cd9c..a452cd3ed 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -253,7 +253,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
#[cfg(not(feature = "tracing"))]
{
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index 3f0f365b2..39c8e0f3d 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -178,7 +178,7 @@ impl<B: Backend> EncodableBlock<B> for Decoder<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
self.embed.encode(state, parameters, command_buffer);
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
index 544532612..910273e7e 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
@@ -1,6 +1,6 @@
//! Full precision embedding lookup encodable.
-use std::rc::Rc;
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -38,7 +38,7 @@ pub enum FullPrecisionEmbeddingLookupError<B: Backend> {
pub struct FullPrecisionEmbeddingLookup<B: Backend> {
kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
- weights_buffer: Rc<B::NativeBuffer>,
+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,
vocab_size: u32,
model_dim: u32,
input_scale: f32,
@@ -78,7 +78,7 @@ impl<B: Backend> FullPrecisionEmbeddingLookup<B> {
Ok(Self {
kernel,
- weights_buffer: weights.buffer_rc(),
+ weights_buffer: weights.buffer(),
vocab_size: vocab_size as u32,
model_dim: model_dim as u32,
input_scale: input_scale.unwrap_or(1.0),
@@ -103,9 +103,9 @@ impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {
let output_array = arrays[1].borrow_mut();
self.kernel.encode(
- token_ids_array.buffer(),
- self.weights_buffer.as_ref(),
- output_array.buffer(),
+ token_ids_array.buffer().borrow().deref(),
+ self.weights_buffer.borrow().deref(),
+ output_array.buffer().borrow().deref(),
batch_size as u32,
self.vocab_size,
self.model_dim,
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
index 8702efb06..7262b584a 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
@@ -1,4 +1,4 @@
-use std::{cell::RefCell, rc::Rc};
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -41,7 +41,7 @@ where
B::Kernels: MatmulKernels,
{
kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
- weights_buffer: Rc<B::NativeBuffer>,
+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,
vocab_size: usize,
model_dim: usize,
}
@@ -89,7 +89,7 @@ where
Ok(Self {
kernel: RefCell::new(kernel),
- weights_buffer: weights.buffer_rc(),
+ weights_buffer: weights.buffer(),
vocab_size,
model_dim,
})
@@ -125,10 +125,10 @@ where
state.context(),
encoder,
FullPrecisionMatmulArguments {
- a: input_array.buffer(),
+ a: input_array.buffer().borrow().deref(),
a_offset: sampling_start * self.model_dim * element_size,
- b: &self.weights_buffer,
- output: output_array.buffer(),
+ b: self.weights_buffer.borrow().deref(),
+ output: output_array.buffer().borrow().deref(),
bias: None,
batch: batch_size,
input_dim: self.model_dim,
diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rs
index be9deebe3..fd9c37509 100644
--- a/crates/uzu/src/encodable_block/full_precision_linear.rs
+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs
@@ -1,4 +1,4 @@
-use std::{cell::RefCell, rc::Rc};
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -49,8 +49,8 @@ where
B::Kernels: MatmulKernels,
{
kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
- bias_buffer: Option<Rc<B::NativeBuffer>>,
- weights_buffer: Rc<B::NativeBuffer>,
+ bias_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,
input_dim: usize,
output_dim: usize,
input_array_id: ArrayId,
@@ -108,7 +108,7 @@ where
});
}
- Some(biases.buffer_rc())
+ Some(biases.buffer())
},
Err(_) => None,
};
@@ -119,7 +119,7 @@ where
Ok(Self {
kernel: RefCell::new(kernel),
bias_buffer,
- weights_buffer: weights.buffer_rc(),
+ weights_buffer: weights.buffer(),
input_dim,
output_dim,
input_array_id,
@@ -147,15 +147,16 @@ where
let input_array = arrays[0].borrow_mut();
let output_array = arrays[1].borrow_mut();
+ let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());
self.kernel.borrow_mut().encode(
state.context(),
encoder,
FullPrecisionMatmulArguments {
- a: input_array.buffer(),
+ a: input_array.buffer().borrow().deref(),
a_offset: 0,
- b: &self.weights_buffer,
- output: output_array.buffer(),
- bias: self.bias_buffer.as_ref().map(|b| b.as_ref()),
+ b: self.weights_buffer.borrow().deref(),
+ output: output_array.buffer().borrow().deref(),
+ bias: bias_borrow.as_deref(),
batch: batch_size,
input_dim: self.input_dim,
output_dim: self.output_dim,
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index dfca55db8..acb6db004 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -264,7 +264,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
// In non-tracing builds, if every sub-block supports shared encoding,
// we can run the entire layer in a single compute encoder.
diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rs
index 866fe03c5..3992419bb 100644
--- a/crates/uzu/src/encodable_block/layer_norm.rs
+++ b/crates/uzu/src/encodable_block/layer_norm.rs
@@ -1,6 +1,6 @@
//! LayerNorm encodable.
-use std::rc::Rc;
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -29,7 +29,7 @@ pub struct LayerNorm<B: Backend> {
config: NormalizationConfig,
input_array_id: ArrayId,
output_array_id: ArrayId,
- scales_buffer: Rc<B::NativeBuffer>,
+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,
}
impl<B: Backend> LayerNorm<B> {
@@ -60,7 +60,7 @@ impl<B: Backend> LayerNorm<B> {
config,
input_array_id,
output_array_id,
- scales_buffer: scales.buffer_rc(),
+ scales_buffer: scales.buffer(),
})
}
}
@@ -96,9 +96,9 @@ impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {
};
self.kernel.encode(
- input_array.buffer(),
- self.scales_buffer.as_ref(),
- output_array.buffer(),
+ input_array.buffer().borrow().deref(),
+ self.scales_buffer.borrow().deref(),
+ output_array.buffer().borrow().deref(),
batch_size,
model_dim,
self.config.epsilon,
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index 45ec30fab..c248bebce 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -1,6 +1,6 @@
//! Mamba2 SSM mixer encodable.
-use std::env;
+use std::{env, ops::Deref};
use crate::{
Activation, DataType,
@@ -177,29 +177,23 @@ impl<B: Backend> MambaMixer<B> {
]);
let in_proj = arrays[0].borrow();
- let input_buf = in_proj.buffer();
-
let conv_inputs = arrays[1].borrow();
- let conv_buf = conv_inputs.buffer();
-
let gate = arrays[2].borrow();
- let gate_buf = gate.buffer();
-
let dt = arrays[3].borrow();
- let dt_buf = dt.buffer();
- let bias_buf = self.gate_bias.buffer();
+ let bias_buf_rc = self.gate_bias.buffer();
+ let bias_buf_borrow = bias_buf_rc.borrow();
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let num_heads = self.config.num_heads;
let total_dim = conv_dim + inner_dim + num_heads;
self.split_inproj.encode(
- input_buf,
- conv_buf,
- gate_buf,
- dt_buf,
- bias_buf,
+ in_proj.buffer().borrow().deref(),
+ conv_inputs.buffer().borrow().deref(),
+ gate.buffer().borrow().deref(),
+ dt.buffer().borrow().deref(),
+ bias_buf_borrow.deref(),
suffix_length as u32,
total_dim as u32,
conv_dim as u32,
@@ -228,14 +222,10 @@ impl<B: Backend> MambaMixer<B> {
let b_arr = arrays[3].borrow();
let c_arr = arrays[4].borrow();
- let input_buf = conv_inputs.buffer();
- let state_buf = conv_state.buffer();
- let x_buf = x_arr.buffer();
- let b_buf = b_arr.buffer();
- let c_buf = c_arr.buffer();
-
- let weight_buf = self.conv_weight.buffer();
- let bias_buf = self.conv_bias.as_ref().map(|arr| arr.buffer());
+ let weight_buf_rc = self.conv_weight.buffer();
+ let weight_buf_borrow = weight_buf_rc.borrow();
+ let bias_buf_rc = self.conv_bias.as_ref().map(|arr| arr.buffer());
+ let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
@@ -245,14 +235,14 @@ impl<B: Backend> MambaMixer<B> {
if suffix_length == 1 {
if conv_dim > 0 && self.config.kernel_size > 0 {
self.conv_decode.encode(
- input_buf,
- weight_buf,
- bias_buf,
- state_buf,
- x_buf,
- b_buf,
- c_buf,
- state_buf,
+ conv_inputs.buffer().borrow().deref(),
+ weight_buf_borrow.deref(),
+ bias_buf_borrow.as_deref(),
+ conv_state.buffer().borrow().deref(),
+ x_arr.buffer().borrow().deref(),
+ b_arr.buffer().borrow().deref(),
+ c_arr.buffer().borrow().deref(),
+ conv_state.buffer().borrow().deref(),
self.config.kernel_size as u32,
conv_dim as u32,
state_stride as u32,
@@ -266,12 +256,12 @@ impl<B: Backend> MambaMixer<B> {
} else {
let padded_buf = if state_stride > 0 {
let array = state.conv_padded_buffer().expect("Missing conv padded buffer");
- let buffer = array.borrow().buffer_rc();
+ let buffer = array.borrow().buffer();
self.conv_pack.encode(
- state_buf,
- input_buf,
- buffer.as_ref(),
+ conv_state.buffer().borrow().deref(),
+ conv_inputs.buffer().borrow().deref(),
+ buffer.borrow().deref(),
state_stride as u32,
conv_dim as u32,
suffix_length as u32,
@@ -285,15 +275,18 @@ impl<B: Backend> MambaMixer<B> {
};
if conv_dim > 0 && self.config.kernel_size > 0 {
- let conv_source = padded_buf.as_ref().map(|b| b.as_ref()).unwrap_or(input_buf);
+ let padded_borrow = padded_buf.as_ref().map(|b| b.borrow());
+ let conv_inputs_borrow = conv_inputs.buffer();
+ let conv_inputs_ref = conv_inputs_borrow.borrow();
+ let conv_source: &B::NativeBuffer = padded_borrow.as_deref().unwrap_or(conv_inputs_ref.deref());
self.conv_scan.encode(
conv_source,
- weight_buf,
- bias_buf,
- x_buf,
- b_buf,
- c_buf,
- state_buf,
+ weight_buf_borrow.deref(),
+ bias_buf_borrow.as_deref(),
+ x_arr.buffer().borrow().deref(),
+ b_arr.buffer().borrow().deref(),
+ c_arr.buffer().borrow().deref(),
+ conv_state.buffer().borrow().deref(),
suffix_length as u32,
self.config.kernel_size as u32,
conv_dim as u32,
@@ -324,38 +317,26 @@ impl<B: Backend> MambaMixer<B> {
]);
let x = base_arrays[0].borrow();
- let x_buf = x.buffer();
-
let b = base_arrays[1].borrow();
- let b_buf = b.buffer();
-
let c = base_arrays[2].borrow();
- let c_buf = c.buffer();
-
let dt = base_arrays[3].borrow();
- let dt_buf = dt.buffer();
-
let z = base_arrays[4].borrow();
- let z_buf = z.buffer();
-
- let state_buf = base_arrays[5].borrow();
- let state_raw = state_buf.buffer();
-
+ let state_arr = base_arrays[5].borrow();
let out = base_arrays[6].borrow();
- let out_buf = out.buffer();
- let skip = self.skip_connection_weight.buffer();
+ let skip_rc = self.skip_connection_weight.buffer();
+ let skip_borrow = skip_rc.borrow();
self.ssd_prefill.encode(
encoder,
SSDPrefillArguments {
- x: x_buf,
- dt: dt_buf,
- b: b_buf,
- c: c_buf,
- d: skip,
- z: z_buf,
- state: state_raw,
- y: out_buf,
+ x: x.buffer().borrow().deref(),
+ dt: dt.buffer().borrow().deref(),
+ b: b.buffer().borrow().deref(),
+ c: c.buffer().borrow().deref(),
+ d: skip_borrow.deref(),
+ z: z.buffer().borrow().deref(),
+ state: state_arr.buffer().borrow().deref(),
+ y: out.buffer().borrow().deref(),
suffix_len: suffix_length,
group_size: (self.config.num_heads / self.config.num_groups) as i32,
state_size: self.config.state_dim as i32,
@@ -386,26 +367,15 @@ impl<B: Backend> MambaMixer<B> {
ArrayId::AttentionOutput,
]);
let x = arrays[0].borrow();
- let x_buf = x.buffer();
-
let b = arrays[1].borrow();
- let b_buf = b.buffer();
-
let c = arrays[2].borrow();
- let c_buf = c.buffer();
-
let dt = arrays[3].borrow();
- let dt_buf = dt.buffer();
-
let z = arrays[4].borrow();
- let z_buf = z.buffer();
-
let state_arr = arrays[5].borrow();
- let state_buf = state_arr.buffer();
-
let y = arrays[6].borrow();
- let y_buf = y.buffer();
- let skip_buf = self.skip_connection_weight.buffer();
+
+ let skip_rc = self.skip_connection_weight.buffer();
+ let skip_borrow = skip_rc.borrow();
let h = self.config.num_heads as u32;
let g = self.config.num_groups as u32;
@@ -420,15 +390,15 @@ impl<B: Backend> MambaMixer<B> {
let state_size = n as i32;
self.ssd_update.encode(
- x_buf,
- dt_buf,
- b_buf,
- c_buf,
- skip_buf,
- z_buf,
- state_buf,
- y_buf,
- state_buf,
+ x.buffer().borrow().deref(),
+ dt.buffer().borrow().deref(),
+ b.buffer().borrow().deref(),
+ c.buffer().borrow().deref(),
+ skip_borrow.deref(),
+ z.buffer().borrow().deref(),
+ state_arr.buffer().borrow().deref(),
+ y.buffer().borrow().deref(),
+ state_arr.buffer().borrow().deref(),
group_size as u32,
state_size as u32,
x_strides.as_slice(),
@@ -469,7 +439,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
if self.supports_shared_encoder() {
command_buffer
diff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rs
index 9a2af53a4..17bf482de 100644
--- a/crates/uzu/src/encodable_block/mlp.rs
+++ b/crates/uzu/src/encodable_block/mlp.rs
@@ -1,5 +1,7 @@
//! MLP block encodable.
+use std::ops::Deref;
+
use crate::{
backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
encodable_block::{EncodableBlock, EncodingParameters},
@@ -31,7 +33,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
&self,
state: &mut ForwardPassState<B>,
params: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
if self.supports_shared_encoder() {
command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, params, encoder));
@@ -45,11 +47,13 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
let fused = arrays[0].borrow_mut();
let hidden = arrays[1].borrow_mut();
let m = fused.shape()[0] as i32;
- let fused_buf = fused.buffer();
- let hidden_buf = hidden.buffer();
+ let fused_buf_rc = fused.buffer();
+ let fused_buf_borrow = fused_buf_rc.borrow();
+ let hidden_buf_rc = hidden.buffer();
+ let hidden_buf_borrow = hidden_buf_rc.borrow();
self.gate
- .encode(encoder, fused_buf, hidden_buf, m)
+ .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref(), m)
.expect("Failed to encode MLP activation/mul kernel");
});
@@ -81,9 +85,13 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
let fused = arrays[0].borrow_mut();
let hidden = arrays[1].borrow_mut();
let m = fused.shape()[0] as i32;
- let fused_buf = fused.buffer();
- let hidden_buf = hidden.buffer();
- self.gate.encode(encoder, fused_buf, hidden_buf, m).expect("Failed to encode MLP activation/mul kernel");
+ let fused_buf_rc = fused.buffer();
+ let fused_buf_borrow = fused_buf_rc.borrow();
+ let hidden_buf_rc = hidden.buffer();
+ let hidden_buf_borrow = hidden_buf_rc.borrow();
+ self.gate
+ .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref(), m)
+ .expect("Failed to encode MLP activation/mul kernel");
drop(fused);
drop(hidden);
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 15bcfa7d9..d1ae53711 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -118,7 +118,7 @@ pub trait EncodableBlock<B: Backend> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
}
diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/moe_block.rs
index 61243d68f..61a4636ae 100644
--- a/crates/uzu/src/encodable_block/moe_block.rs
+++ b/crates/uzu/src/encodable_block/moe_block.rs
@@ -1,6 +1,10 @@
//! MoE (Mixture of Experts) block encodable.
-use std::{cell::RefCell, rc::Rc};
+use std::{
+ cell::RefCell,
+ ops::{Deref, DerefMut},
+ rc::Rc,
+};
use thiserror::Error;
@@ -24,16 +28,16 @@ use crate::{
};
struct RouterBlock<B: Backend> {
- weights_buf: Rc<B::NativeBuffer>,
- biases_buf: Rc<B::NativeBuffer>,
+ weights_buf: Rc<RefCell<B::NativeBuffer>>,
+ biases_buf: Rc<RefCell<B::NativeBuffer>>,
}
#[derive(Clone)]
struct SharedMoeWeights<B: Backend> {
- pub w13_buf: Rc<B::NativeBuffer>,
- pub w2_buf: Rc<B::NativeBuffer>,
- pub up_biases_buf: Rc<B::NativeBuffer>,
- pub down_biases_buf: Rc<B::NativeBuffer>,
+ pub w13_buf: Rc<RefCell<B::NativeBuffer>>,
+ pub w2_buf: Rc<RefCell<B::NativeBuffer>>,
+ pub up_biases_buf: Rc<RefCell<B::NativeBuffer>>,
+ pub down_biases_buf: Rc<RefCell<B::NativeBuffer>>,
}
pub struct MoeBlock<B: Backend> {
@@ -91,8 +95,8 @@ impl<B: Backend> MoeBlock<B> {
let weights_arr = router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?;
let biases_arr = router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?;
RouterBlock {
- weights_buf: weights_arr.buffer_rc(),
- biases_buf: biases_arr.buffer_rc(),
+ weights_buf: weights_arr.buffer(),
+ biases_buf: biases_arr.buffer(),
}
},
LinearConfig::QLoRA {
@@ -146,10 +150,10 @@ impl<B: Backend> MoeBlock<B> {
.map_err(MoeBlockError::ParameterLoaderError)?;
let shared_weights = SharedMoeWeights {
- w13_buf: w13_arr.buffer_rc(),
- w2_buf: w2_arr.buffer_rc(),
- up_biases_buf: up_biases_arr.buffer_rc(),
- down_biases_buf: down_biases_arr.buffer_rc(),
+ w13_buf: w13_arr.buffer(),
+ w2_buf: w2_arr.buffer(),
+ up_biases_buf: up_biases_arr.buffer(),
+ down_biases_buf: down_biases_arr.buffer(),
};
Ok(Self {
@@ -189,7 +193,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
let suffix_length = state.active_suffix_length();
let arrays = state.arrays(&[
@@ -215,7 +219,7 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
ArrayId::MoeTwoPassRowExpertMap,
]);
- let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<B::NativeBuffer> { array.borrow().buffer_rc() };
+ let clone_buffer = |array: &RefCell<Array<B>>| -> Rc<RefCell<B::NativeBuffer>> { array.borrow().buffer() };
let mut array_iter = arrays.iter();
let main_buf = clone_buffer(array_iter.next().unwrap());
@@ -242,12 +246,10 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
let e = self.moe_config.num_routed_experts;
let k = self.moe_config.num_active_routed_experts;
-
- let root = command_buffer;
let k_tile = 128;
// Clear internal MoE buffers
- root.with_copy_encoder(|encoder| {
+ command_buffer.with_copy_encoder(|encoder| {
if suffix_length > 0 && k > 0 {
let entries = suffix_length * k;
let topk_bytes = entries * std::mem::size_of::<u32>();
@@ -255,28 +257,28 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
// Clear topk_ids and tok2row buffers
if topk_bytes > 0 {
- encoder.encode_fill(&topk_ids_buf, 0..topk_bytes, 0xFF);
+ encoder.encode_fill(topk_ids_buf.borrow_mut().deref_mut(), 0..topk_bytes, 0xFF);
}
if tok2row_bytes > 0 {
- encoder.encode_fill(&tok2row_buf, 0..tok2row_bytes, 0xFF);
+ encoder.encode_fill(tok2row_buf.borrow_mut().deref_mut(), 0..tok2row_bytes, 0xFF);
}
// Clear hidden buffer
let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
if hidden_bytes > 0 {
- encoder.encode_fill(&hidden_buf, 0..hidden_bytes, 0);
+ encoder.encode_fill(hidden_buf.borrow_mut().deref_mut(), 0..hidden_bytes, 0);
}
// Clear y_partial buffer
let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
if y_partial_bytes > 0 {
- encoder.encode_fill(&y_partial_buf, 0..y_partial_bytes, 0);
+ encoder.encode_fill(y_partial_buf.borrow_mut().deref_mut(), 0..y_partial_bytes, 0);
}
// Clear x_perm buffer
let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
if x_perm_bytes > 0 {
- encoder.encode_fill(&x_perm_buf, 0..x_perm_bytes, 0);
+ encoder.encode_fill(x_perm_buf.borrow_mut().deref_mut(), 0..x_perm_bytes, 0);
}
}
});
@@ -296,11 +298,11 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
command_buffer.with_compute_encoder(|encoder| {
// Use the fused router+topk kernel
self.router_topk_kernel.encode(
- main_buf.as_ref(),
- self.router.weights_buf.as_ref(),
- self.router.biases_buf.as_ref(),
- topk_ids_buf.as_ref(),
- topk_probs_buf.as_ref(),
+ main_buf.borrow().deref(),
+ self.router.weights_buf.borrow().deref(),
+ self.router.biases_buf.borrow().deref(),
+ topk_ids_buf.borrow().deref(),
+ topk_probs_buf.borrow().deref(),
suffix_length as u32,
self.model_dim as u32,
e as u32,
@@ -313,10 +315,10 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
command_buffer.with_compute_encoder(|encoder| {
self.counts_offsets_kernel.encode(
- topk_ids_buf.as_ref(),
- offsets_buf.as_ref(),
- sumk_buf.as_ref(),
- partials_buf.as_ref(),
+ topk_ids_buf.borrow().deref(),
+ offsets_buf.borrow().deref(),
+ sumk_buf.borrow().deref(),
+ partials_buf.borrow().deref(),
suffix_length as u32,
e as u32,
k as u32,
@@ -328,9 +330,9 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
let num_tiles = ((e + 512 - 1) / 512).max(1);
command_buffer.with_compute_encoder(|encoder| {
self.scatter_bases_kernel.encode(
- partials_buf.as_ref(),
- block_bases_buf.as_ref(),
- block_alloc_buf.as_ref(),
+ partials_buf.borrow().deref(),
+ block_bases_buf.borrow().deref(),
+ block_alloc_buf.borrow().deref(),
e as u32,
num_blocks as u32,
num_tiles as u32,
@@ -340,31 +342,31 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
});
command_buffer.with_compute_encoder(|encoder| {
self.scatter_map_kernel.encode(
- topk_ids_buf.as_ref(),
- topk_probs_buf.as_ref(),
- offsets_buf.as_ref(),
- block_bases_buf.as_ref(),
- block_alloc_buf.as_ref(),
- bucketed_ids_buf.as_ref(),
- bucketed_probs_buf.as_ref(),
+ topk_ids_buf.borrow().deref(),
+ topk_probs_buf.borrow().deref(),
+ offsets_buf.borrow().deref(),
+ block_bases_buf.borrow().deref(),
+ block_alloc_buf.borrow().deref(),
+ bucketed_ids_buf.borrow().deref(),
+ bucketed_probs_buf.borrow().deref(),
suffix_length as u32,
e as u32,
k as u32,
num_blocks as u32,
num_tiles as u32,
- tok2row_buf.as_ref(),
+ tok2row_buf.borrow().deref(),
&encoder,
);
});
self.gather_kernels.encode(
- root,
+ command_buffer,
self.data_type,
&MoeGatherArguments {
- x_buffer: &main_buf,
- bucketed_ids_buffer: &bucketed_ids_buf,
- x_perm_buffer: &x_perm_buf,
- sumk_buffer: &sumk_buf,
+ x_buffer: main_buf.borrow().deref(),
+ bucketed_ids_buffer: bucketed_ids_buf.borrow().deref(),
+ x_perm_buffer: x_perm_buf.borrow().deref(),
+ sumk_buffer: sumk_buf.borrow().deref(),
t: suffix_length,
k,
d_model: self.model_dim,
@@ -385,22 +387,22 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
self.experts_two_pass_decode_kernel.encode(
- root,
+ command_buffer,
&MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &y_partial_buf,
- w13_all: &self.shared_weights.w13_buf,
- w2_all: &self.shared_weights.w2_buf,
- up_biases: &self.shared_weights.up_biases_buf,
- down_biases: &self.shared_weights.down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
+ x_perm_buffer: x_perm_buf.borrow().deref(),
+ expert_offsets: offsets_buf.borrow().deref(),
+ row_expert_map: row_expert_map_buf.borrow().deref(),
+ hidden_buffer: hidden_buf.borrow_mut().deref_mut(),
+ output_buffer: y_partial_buf.borrow().deref(),
+ w13_all: self.shared_weights.w13_buf.borrow().deref(),
+ w2_all: self.shared_weights.w2_buf.borrow().deref(),
+ up_biases: self.shared_weights.up_biases_buf.borrow().deref(),
+ down_biases: self.shared_weights.down_biases_buf.borrow().deref(),
+ tile_counts: tile_counts_buf.borrow().deref(),
+ tile_offsets: tile_offsets_buf.borrow().deref(),
+ tile_map: tile_map_buf.borrow().deref(),
+ total_tiles: total_tiles_buf.borrow().deref(),
+ dispatch_args: dispatch_args_buf.borrow().deref(),
total_rows,
d_model: self.model_dim,
d_ff: self.hidden_dim,
@@ -419,21 +421,36 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
let total_rows = suffix_length * k;
let num_tiles_k = ((self.hidden_dim + k_tile - 1) / k_tile) as u32;
- let args = MoeExpertsTwoPassArguments {
- x_perm_buffer: x_perm_buf.as_ref(),
- expert_offsets: offsets_buf.as_ref(),
- row_expert_map: row_expert_map_buf.as_ref(),
- hidden_buffer: hidden_buf.as_ref(),
- output_buffer: y_partial_buf.as_ref(),
- w13_all: self.shared_weights.w13_buf.as_ref(),
- w2_all: self.shared_weights.w2_buf.as_ref(),
- up_biases: self.shared_weights.up_biases_buf.as_ref(),
- down_biases: self.shared_weights.down_biases_buf.as_ref(),
- tile_counts: tile_counts_buf.as_ref(),
- tile_offsets: tile_offsets_buf.as_ref(),
- tile_map: tile_map_buf.as_ref(),
- total_tiles: total_tiles_buf.as_ref(),
- dispatch_args: dispatch_args_buf.as_ref(),
+ let x_perm_borrow = x_perm_buf.borrow();
+ let offsets_borrow = offsets_buf.borrow();
+ let row_expert_map_borrow = row_expert_map_buf.borrow();
+ let mut hidden_borrow = hidden_buf.borrow_mut();
+ let y_partial_borrow = y_partial_buf.borrow();
+ let w13_borrow = self.shared_weights.w13_buf.borrow();
+ let w2_borrow = self.shared_weights.w2_buf.borrow();
+ let up_biases_borrow = self.shared_weights.up_biases_buf.borrow();
+ let down_biases_borrow = self.shared_weights.down_biases_buf.borrow();
+ let tile_counts_borrow = tile_counts_buf.borrow();
+ let tile_offsets_borrow = tile_offsets_buf.borrow();
+ let tile_map_borrow = tile_map_buf.borrow();
+ let total_tiles_borrow = total_tiles_buf.borrow();
+ let dispatch_args_borrow = dispatch_args_buf.borrow();
+
+ let mut args = MoeExpertsTwoPassArguments {
+ x_perm_buffer: x_perm_borrow.deref(),
+ expert_offsets: offsets_borrow.deref(),
+ row_expert_map: row_expert_map_borrow.deref(),
+ hidden_buffer: hidden_borrow.deref_mut(),
+ output_buffer: y_partial_borrow.deref(),
+ w13_all: w13_borrow.deref(),
+ w2_all: w2_borrow.deref(),
+ up_biases: up_biases_borrow.deref(),
+ down_biases: down_biases_borrow.deref(),
+ tile_counts: tile_counts_borrow.deref(),
+ tile_offsets: tile_offsets_borrow.deref(),
+ tile_map: tile_map_borrow.deref(),
+ total_tiles: total_tiles_borrow.deref(),
+ dispatch_args: dispatch_args_borrow.deref(),
total_rows,
d_model: self.model_dim,
d_ff: self.hidden_dim,
@@ -447,15 +464,15 @@ impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
silu_alpha,
data_type: self.data_type,
};
- self.experts_two_pass_prefill_kernel.encode(root, &args);
+ self.experts_two_pass_prefill_kernel.encode(command_buffer, &mut args);
}
command_buffer.with_compute_encoder(|encoder| {
self.finalize_kernel.encode(
- tok2row_buf.as_ref(),
- topk_probs_buf.as_ref(),
- y_partial_buf.as_ref(),
- main_buf.as_ref(),
+ tok2row_buf.borrow().deref(),
+ topk_probs_buf.borrow().deref(),
+ y_partial_buf.borrow().deref(),
+ main_buf.borrow().deref(),
suffix_length as u32,
self.model_dim as u32,
k as u32,
diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rs
index 3ae7a4b14..1424c6013 100644
--- a/crates/uzu/src/encodable_block/pooling.rs
+++ b/crates/uzu/src/encodable_block/pooling.rs
@@ -1,5 +1,7 @@
//! Pooling encodable for sequence-level aggregation.
+use std::ops::Deref;
+
use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
@@ -63,7 +65,7 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
@@ -92,8 +94,8 @@ impl<B: Backend> EncodableBlock<B> for Pooling<B> {
let pooling_array = arrays[1].borrow_mut();
self.pooling_kernel.encode(
- main_array.buffer(),
- pooling_array.buffer(),
+ main_array.buffer().borrow().deref(),
+ pooling_array.buffer().borrow().deref(),
seq_len as u32,
self.model_dim as u32,
batch_size,
diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rs
index fb3b12d61..6cbca8760 100644
--- a/crates/uzu/src/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/encodable_block/prediction_head.rs
@@ -40,7 +40,7 @@ impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
if self.supports_shared_encoder() {
command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));
diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
index 501930eaa..4d51f0932 100644
--- a/crates/uzu/src/encodable_block/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -1,6 +1,6 @@
//! QK Normalization encodable.
-use std::rc::Rc;
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -30,8 +30,8 @@ pub struct QKNorm<B: Backend> {
query_config: Option<NormalizationConfig>,
key_config: Option<NormalizationConfig>,
qkv_array_id: ArrayId,
- query_scales_buffer: Option<Rc<B::NativeBuffer>>,
- key_scales_buffer: Option<Rc<B::NativeBuffer>>,
+ query_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
+ key_scales_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
num_q_heads: usize,
num_kv_heads: usize,
head_dim: usize,
@@ -75,7 +75,7 @@ impl<B: Backend> QKNorm<B> {
.map_err(QKNormError::BackendError)?;
query_kernel = Some(kernel);
- query_scales_buffer = Some(scales.buffer_rc());
+ query_scales_buffer = Some(scales.buffer());
}
// Setup key normalization if configured
@@ -99,7 +99,7 @@ impl<B: Backend> QKNorm<B> {
.map_err(QKNormError::BackendError)?;
key_kernel = Some(kernel);
- key_scales_buffer = Some(scales.buffer_rc());
+ key_scales_buffer = Some(scales.buffer());
}
Ok(Self {
@@ -135,7 +135,6 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
};
let qkv_array = qkv_binding[0].borrow_mut();
- let qkv_buffer = qkv_array.buffer();
let batch_size = qkv_shape[0] as u32;
// Process query normalization if configured
@@ -143,9 +142,9 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
(&self.query_kernel, &self.query_scales_buffer, &self.query_config)
{
query_kernel.encode(
- qkv_buffer,
- query_scales_buffer.as_ref(),
- qkv_buffer,
+ qkv_array.buffer().borrow().deref(),
+ query_scales_buffer.borrow().deref(),
+ qkv_array.buffer().borrow().deref(),
batch_size,
self.num_q_heads as u32,
self.num_kv_heads as u32,
@@ -164,9 +163,9 @@ impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
(&self.key_kernel, &self.key_scales_buffer, &self.key_config)
{
key_kernel.encode(
- qkv_buffer,
- key_scales_buffer.as_ref(),
- qkv_buffer,
+ qkv_array.buffer().borrow().deref(),
+ key_scales_buffer.borrow().deref(),
+ qkv_array.buffer().borrow().deref(),
batch_size,
self.num_q_heads as u32,
self.num_kv_heads as u32,
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
index dcfab22bf..7d27262f4 100644
--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
+++ b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
@@ -1,6 +1,6 @@
//! Quantized embedding lookup encodable.
-use std::rc::Rc;
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -67,9 +67,9 @@ pub enum QuantizedEmbeddingLookupError<B: Backend> {
pub struct QuantizedEmbeddingLookup<B: Backend> {
kernel: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
- weights_buffer: Rc<B::NativeBuffer>,
- scales_buffer: Rc<B::NativeBuffer>,
- biases_buffer: Rc<B::NativeBuffer>,
+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,
+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,
+ biases_buffer: Rc<RefCell<B::NativeBuffer>>,
mode: QuantizationMode,
input_scale: f32,
vocab_size: u32,
@@ -198,7 +198,7 @@ impl<B: Backend> QuantizedEmbeddingLookup<B> {
});
}
- biases.buffer_rc()
+ biases.buffer()
},
Err(_) => {
let element_size = match data_type {
@@ -216,14 +216,14 @@ impl<B: Backend> QuantizedEmbeddingLookup<B> {
std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);
}
- Rc::new(buffer)
+ Rc::new(RefCell::new(buffer))
},
};
Ok(Self {
kernel,
- weights_buffer: weights.buffer_rc(),
- scales_buffer: scales.buffer_rc(),
+ weights_buffer: weights.buffer(),
+ scales_buffer: scales.buffer(),
biases_buffer,
mode,
input_scale,
@@ -257,11 +257,11 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {
};
self.kernel.encode(
- token_ids_array.buffer(),
- self.weights_buffer.as_ref(),
- self.scales_buffer.as_ref(),
- self.biases_buffer.as_ref(),
- output_array.buffer(),
+ token_ids_array.buffer().borrow().deref(),
+ self.weights_buffer.borrow().deref(),
+ self.scales_buffer.borrow().deref(),
+ self.biases_buffer.borrow().deref(),
+ output_array.buffer().borrow().deref(),
batch_size as u32,
self.vocab_size,
self.model_dim,
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
index 4d13c1da8..1576d8d8c 100644
--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
@@ -1,6 +1,6 @@
//! Quantized embedding readout encodable.
-use std::rc::Rc;
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -67,9 +67,9 @@ pub enum QuantizedEmbeddingReadoutError<B: Backend> {
pub struct QuantizedEmbeddingReadout<B: Backend> {
kernel: QuantizedMatmulKernelEncodable<B>,
- weights_buffer: Rc<B::NativeBuffer>,
- scales_buffer: Rc<B::NativeBuffer>,
- biases_buffer: Rc<B::NativeBuffer>,
+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,
+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,
+ biases_buffer: Rc<RefCell<B::NativeBuffer>>,
vocab_size: usize,
model_dim: usize,
}
@@ -178,7 +178,7 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {
got: deq_biases.data_type(),
});
}
- deq_biases.buffer_rc()
+ deq_biases.buffer()
},
Err(_) => {
let element_size = match data_type {
@@ -195,7 +195,7 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {
std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);
}
- Rc::new(buffer)
+ Rc::new(RefCell::new(buffer))
},
};
@@ -215,8 +215,8 @@ impl<B: Backend> QuantizedEmbeddingReadout<B> {
Ok(Self {
kernel,
- weights_buffer: weights.buffer_rc(),
- scales_buffer: scales.buffer_rc(),
+ weights_buffer: weights.buffer(),
+ scales_buffer: scales.buffer(),
biases_buffer,
vocab_size,
model_dim,
@@ -252,12 +252,12 @@ impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {
.encode(
encoder,
QuantizedMatmulArguments {
- a_buffer: input_array.buffer(),
+ a_buffer: input_array.buffer().borrow().deref(),
a_offset,
- b_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- zero_points_or_biases_buffer: &self.biases_buffer,
- output_buffer: output_array.buffer(),
+ b_buffer: self.weights_buffer.borrow().deref(),
+ scales_buffer: self.scales_buffer.borrow().deref(),
+ zero_points_or_biases_buffer: self.biases_buffer.borrow().deref(),
+ output_buffer: output_array.buffer().borrow().deref(),
batch: batch_size,
input_dim: self.model_dim,
output_dim: self.vocab_size,
diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rs
index 4cffe7a56..fa8f4fdd4 100644
--- a/crates/uzu/src/encodable_block/quantized_linear.rs
+++ b/crates/uzu/src/encodable_block/quantized_linear.rs
@@ -1,4 +1,4 @@
-use std::rc::Rc;
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -84,10 +84,10 @@ pub enum QuantizedLinearError<B: Backend> {
pub struct QuantizedLinear<B: Backend> {
kernel: QuantizedMatmulKernelEncodable<B>,
bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
- biases_buffer: Option<Rc<B::NativeBuffer>>,
- weights_buffer: Rc<B::NativeBuffer>,
- scales_buffer: Rc<B::NativeBuffer>,
- zero_points_or_biases_buffer: Rc<B::NativeBuffer>,
+ biases_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
+ weights_buffer: Rc<RefCell<B::NativeBuffer>>,
+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,
+ zero_points_or_biases_buffer: Rc<RefCell<B::NativeBuffer>>,
quantization_type: QuantizedMatmulType,
input_dim: usize,
output_dim: usize,
@@ -155,7 +155,7 @@ impl<B: Backend> QuantizedLinear<B> {
});
}
- (QuantizedMatmulType::Mlx, deq_biases.buffer_rc())
+ (QuantizedMatmulType::Mlx, deq_biases.buffer())
},
Err(_) => {
let zero_points = parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;
@@ -181,7 +181,7 @@ impl<B: Backend> QuantizedLinear<B> {
});
}
- (QuantizedMatmulType::ZeroPoint, zero_points.buffer_rc())
+ (QuantizedMatmulType::ZeroPoint, zero_points.buffer())
},
};
@@ -204,7 +204,7 @@ impl<B: Backend> QuantizedLinear<B> {
let bias_add_kernel = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type)
.map_err(QuantizedLinearError::BackendError)?;
- (Some(bias_add_kernel), Some(biases.buffer_rc()))
+ (Some(bias_add_kernel), Some(biases.buffer()))
},
Err(_) => (None, None),
};
@@ -227,8 +227,8 @@ impl<B: Backend> QuantizedLinear<B> {
kernel,
bias_add_kernel,
biases_buffer,
- weights_buffer: weights.buffer_rc(),
- scales_buffer: scales.buffer_rc(),
+ weights_buffer: weights.buffer(),
+ scales_buffer: scales.buffer(),
zero_points_or_biases_buffer,
quantization_type,
input_dim,
@@ -254,18 +254,19 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {
let batch_size = state.active_suffix_length();
let input_array = arrays[0].borrow_mut();
let output_array = arrays[1].borrow_mut();
- let output_buffer = output_array.buffer();
+ let output_buf_rc = output_array.buffer();
+ let output_buf_borrow = output_buf_rc.borrow();
self.kernel
.encode(
encoder,
QuantizedMatmulArguments {
- a_buffer: input_array.buffer(),
+ a_buffer: input_array.buffer().borrow().deref(),
a_offset: 0,
- b_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,
- output_buffer,
+ b_buffer: self.weights_buffer.borrow().deref(),
+ scales_buffer: self.scales_buffer.borrow().deref(),
+ zero_points_or_biases_buffer: self.zero_points_or_biases_buffer.borrow().deref(),
+ output_buffer: output_buf_borrow.deref(),
batch: batch_size,
input_dim: self.input_dim,
output_dim: self.output_dim,
@@ -277,9 +278,9 @@ impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {
if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {
let total_length = batch_size * self.output_dim;
bias_add_kernel.encode_if(
- output_buffer,
- biases_buffer.as_ref(),
- output_buffer,
+ output_buf_borrow.deref(),
+ biases_buffer.borrow().deref(),
+ output_buf_borrow.deref(),
self.output_dim as u32,
total_length as u32,
encoder,
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 1c0a18050..a1b7cea4c 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -1,6 +1,6 @@
//! RMS Normalization encodable.
-use std::rc::Rc;
+use std::{cell::RefCell, ops::Deref, rc::Rc};
use thiserror::Error;
@@ -29,7 +29,7 @@ pub struct RMSNorm<B: Backend> {
config: NormalizationConfig,
input_array_id: ArrayId,
output_array_id: ArrayId,
- scales_buffer: Rc<B::NativeBuffer>,
+ scales_buffer: Rc<RefCell<B::NativeBuffer>>,
use_sampling_range: bool,
}
@@ -66,7 +66,7 @@ impl<B: Backend> RMSNorm<B> {
config,
input_array_id,
output_array_id,
- scales_buffer: scales.buffer_rc(),
+ scales_buffer: scales.buffer(),
use_sampling_range: false,
})
}
@@ -122,9 +122,9 @@ impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {
let output_offset = batch_start * output_row_size_in_bytes;
self.kernel.encode(
- (input_array.buffer(), input_offset),
- self.scales_buffer.as_ref(),
- (output_array.buffer(), output_offset),
+ (input_array.buffer().borrow().deref(), input_offset),
+ self.scales_buffer.borrow().deref(),
+ (output_array.buffer().borrow().deref(), output_offset),
batch_len as u32,
input_shape[1] as u32,
self.config.epsilon,
diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rs
index c54c13876..2cbd65be8 100644
--- a/crates/uzu/src/encodable_block/rope.rs
+++ b/crates/uzu/src/encodable_block/rope.rs
@@ -1,5 +1,7 @@
//! Rope (Rotary Position Embedding) encodable.
+use std::ops::Deref;
+
use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
@@ -82,12 +84,12 @@ impl<B: Backend> EncodableBlock<B> for Rope<B> {
let token_positions_offset = token_positions.offset();
self.kernel.encode(
- qkv.buffer(),
- rope_cosines.buffer(),
- rope_sines.buffer(),
- (token_positions.buffer(), token_positions_offset),
- rotated_queries.buffer(),
- rotated_keys.buffer(),
+ qkv.buffer().borrow().deref(),
+ rope_cosines.buffer().borrow().deref(),
+ rope_sines.buffer().borrow().deref(),
+ (token_positions.buffer().borrow().deref(), token_positions_offset),
+ rotated_queries.buffer().borrow().deref(),
+ rotated_keys.buffer().borrow().deref(),
head_dim as u32,
num_heads as u32,
num_groups as u32,
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index 6e1609c15..e3644d572 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -1,5 +1,7 @@
//! Sampling kernel encodable.
+use std::ops::Deref;
+
use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
@@ -74,15 +76,22 @@ impl<B: Backend> EncodableBlock<B> for Sampling<B> {
let bitmask = cell.borrow();
let bitmask_row_len = bitmask.shape()[1];
let bitmask_offset = bitmask.offset() + sampling_start * bitmask_row_len * std::mem::size_of::<u32>();
- (Some(bitmask.buffer_rc()), bitmask_offset)
+ (Some(bitmask.buffer()), bitmask_offset)
});
+ let bitmask_borrow = bitmask_buffer.as_ref().map(|b| b.borrow());
+ let logits_buf_rc = logits.buffer();
+ let logits_buf_borrow = logits_buf_rc.borrow();
+ let seeds_buf_rc = seeds.buffer();
+ let seeds_buf_borrow = seeds_buf_rc.borrow();
+ let output_buf_rc = output_buffer_ref.buffer();
+ let output_buf_borrow = output_buf_rc.borrow();
if let Err(e) = self.kernel.encode_with_encoder(
- logits.buffer(),
- Some(seeds.buffer()),
+ logits_buf_borrow.deref(),
+ Some(seeds_buf_borrow.deref()),
seeds_offset,
- bitmask_buffer.as_ref().map(|b| b.as_ref()),
+ bitmask_borrow.as_deref(),
bitmask_offset,
- output_buffer_ref.buffer(),
+ output_buf_borrow.deref(),
sampling_method,
batch_size,
vocab_size,
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index 6609366fe..6b79f4437 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -1,3 +1,5 @@
+use std::ops::Deref;
+
use crate::{
DataType,
array::Array,
@@ -111,7 +113,7 @@ impl<B: Backend> ShortConvMixer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
let active_suffix_length = state.active_suffix_length();
if active_suffix_length == 0 {
@@ -229,12 +231,10 @@ impl<B: Backend> ShortConvMixer<B> {
let conv_state = arrays[1].borrow_mut();
let out = arrays[2].borrow_mut();
- let in_proj_buf = in_proj.buffer();
- let state_buf = conv_state.buffer();
- let out_buf = out.buffer();
-
- let weight_buf = self.conv_weight.buffer();
- let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());
+ let weight_buf_rc = self.conv_weight.buffer();
+ let weight_buf_borrow = weight_buf_rc.borrow();
+ let bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
+ let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
@@ -246,8 +246,8 @@ impl<B: Backend> ShortConvMixer<B> {
let padded_size = padded_rows * self.model_dim * element_size;
let padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");
self.short_conv_pack.encode(
- state_buf,
- in_proj_buf,
+ conv_state.buffer().borrow().deref(),
+ in_proj.buffer().borrow().deref(),
&padded_buf,
state_stride as u32,
suffix_length as u32,
@@ -258,11 +258,11 @@ impl<B: Backend> ShortConvMixer<B> {
self.short_conv_prefill.encode(
&padded_buf,
- in_proj_buf,
- weight_buf,
- bias_buf,
- out_buf,
- state_buf,
+ in_proj.buffer().borrow().deref(),
+ weight_buf_borrow.deref(),
+ bias_buf_borrow.as_deref(),
+ out.buffer().borrow().deref(),
+ conv_state.buffer().borrow().deref(),
suffix_length as u32,
kernel_size as u32,
self.model_dim as u32 * 3,
@@ -309,16 +309,19 @@ impl<B: Backend> ShortConvMixer<B> {
let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;
let base_state_offset = conv_state.offset();
let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();
- let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());
+ let trie_weight_buf_rc = self.conv_weight.buffer();
+ let trie_weight_buf_borrow = trie_weight_buf_rc.borrow();
+ let trie_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
+ let trie_bias_buf_borrow = trie_bias_buf_rc.as_ref().map(|rc| rc.borrow());
self.short_conv_trie.encode(
- (in_proj.buffer(), in_proj_offset),
- self.conv_weight.buffer(),
- bias_buf,
- (conv_state.buffer(), base_state_offset),
- (parents.buffer(), parents_offset),
- (out.buffer(), out_offset),
- (suffix_state.buffer(), suffix_state_offset),
+ (in_proj.buffer().borrow().deref(), in_proj_offset),
+ trie_weight_buf_borrow.deref(),
+ trie_bias_buf_borrow.as_deref(),
+ (conv_state.buffer().borrow().deref(), base_state_offset),
+ (parents.buffer().borrow().deref(), parents_offset),
+ (out.buffer().borrow().deref(), out_offset),
+ (suffix_state.buffer().borrow().deref(), suffix_state_offset),
trie_len as u32,
kernel_size as u32,
in_proj_stride as u32,
@@ -344,17 +347,20 @@ impl<B: Backend> ShortConvMixer<B> {
let conv_state = arrays[1].borrow_mut();
let out = arrays[2].borrow_mut();
- let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());
+ let decode_weight_buf_rc = self.conv_weight.buffer();
+ let decode_weight_buf_borrow = decode_weight_buf_rc.borrow();
+ let decode_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
+ let decode_bias_buf_borrow = decode_bias_buf_rc.as_ref().map(|rc| rc.borrow());
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
self.short_conv_decode.encode(
- in_proj.buffer(),
- self.conv_weight.buffer(),
- bias_buf,
- conv_state.buffer(),
- out.buffer(),
- conv_state.buffer(),
+ in_proj.buffer().borrow().deref(),
+ decode_weight_buf_borrow.deref(),
+ decode_bias_buf_borrow.as_deref(),
+ conv_state.buffer().borrow().deref(),
+ out.buffer().borrow().deref(),
+ conv_state.buffer().borrow().deref(),
suffix_length as u32,
kernel_size as u32,
self.model_dim as u32 * 3,
@@ -370,7 +376,7 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
) {
if self.supports_shared_encoder() {
command_buffer
diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rs
index da725f0b5..9aa6ba3da 100644
--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs
@@ -1,5 +1,7 @@
//! Tensor add-swap encodable.
+use std::ops::Deref;
+
use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
@@ -48,6 +50,11 @@ impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {
let skip_array = arrays[0].borrow_mut();
let main_array = arrays[1].borrow_mut();
- self.kernel.encode(skip_array.buffer(), main_array.buffer(), length as u32, encoder);
+ self.kernel.encode(
+ skip_array.buffer().borrow().deref(),
+ main_array.buffer().borrow().deref(),
+ length as u32,
+ encoder,
+ );
}
}
diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rs
index d6d230fb3..211c398a1 100644
--- a/crates/uzu/src/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/encodable_block/tensor_copy.rs
@@ -1,5 +1,7 @@
//! Tensor copy encodable.
+use std::ops::Deref;
+
use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
@@ -48,6 +50,11 @@ impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {
let source_array = arrays[0].borrow_mut();
let destination_array = arrays[1].borrow_mut();
- self.kernel.encode(source_array.buffer(), destination_array.buffer(), length as u32, encoder);
+ self.kernel.encode(
+ source_array.buffer().borrow().deref(),
+ destination_array.buffer().borrow().deref(),
+ length as u32,
+ encoder,
+ );
}
}
diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
index 87d8b82df..e3485cd81 100644
--- a/crates/uzu/src/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -276,7 +276,7 @@ impl<B: Backend> CacheLayers<B> {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
kv_cache_update: &KVCacheUpdate<B>,
) {
let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);
diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rs
index 9074accdf..def9925c7 100644
--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs
@@ -163,7 +163,7 @@ impl<B: Backend> KVCacheLayer<B> {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
kv_cache_update: &KVCacheUpdate<B>,
) {
match &mut self.state {
@@ -213,7 +213,7 @@ impl<B: Backend> KVCacheLayer<B> {
&self,
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
kv_cache_update: &KVCacheUpdate<B>,
) {
if source_indices == destination_indices {
@@ -224,9 +224,9 @@ impl<B: Backend> KVCacheLayer<B> {
let v_shape = self.values.borrow().shape().to_vec();
let layer_data = KVLayerData {
- key_buffer: self.keys.borrow().buffer_rc(),
+ key_buffer: self.keys.borrow().buffer(),
key_shape: [k_shape[0], k_shape[1], k_shape[2]],
- value_buffer: self.values.borrow().buffer_rc(),
+ value_buffer: self.values.borrow().buffer(),
value_shape: [v_shape[0], v_shape[1], v_shape[2]],
};
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index c61b7ea32..b78a8c777 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -1,4 +1,9 @@
-use std::{cell::RefCell, collections::HashMap, rc::Rc};
+use std::{
+ cell::RefCell,
+ collections::HashMap,
+ ops::{Deref, DerefMut},
+ rc::Rc,
+};
#[cfg(feature = "tracing")]
use crate::forward_pass::traces::ActivationTrace;
@@ -152,8 +157,8 @@ impl<B: Backend> ForwardPassState<B> {
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
should_fill_attention_bias: bool,
- async_positions: Option<(Rc<B::NativeBuffer>, usize)>,
- async_seeds: Option<(Rc<B::NativeBuffer>, usize)>,
+ async_positions: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,
+ async_seeds: Option<(Rc<RefCell<B::NativeBuffer>>, usize)>,
) -> Self {
let suffix_length = token_ids.len();
assert_eq!(suffix_length, token_positions.len(), "Tokens and positions must have same length");
@@ -387,7 +392,9 @@ impl<B: Backend> ForwardPassState<B> {
let create_buffer = |size: usize| -> ArrayCell<B> {
let buffer_size = size * data_type.size_in_bytes();
let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- RefCell::new(unsafe { Array::from_parts(Rc::new(buffer), 0, &[batch_size, size / batch_size], data_type) })
+ RefCell::new(unsafe {
+ Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, size / batch_size], data_type)
+ })
};
ClassifierModeState {
@@ -397,7 +404,9 @@ impl<B: Backend> ForwardPassState<B> {
classifier_logits: {
let buffer_size = batch_size * num_labels * data_type.size_in_bytes();
let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- RefCell::new(unsafe { Array::from_parts(Rc::new(buffer), 0, &[batch_size, num_labels], data_type) })
+ RefCell::new(unsafe {
+ Array::from_parts(Rc::new(RefCell::new(buffer)), 0, &[batch_size, num_labels], data_type)
+ })
},
traces: Rc::new(RefCell::new(ActivationTrace::new_classifier(
context,
@@ -423,7 +432,7 @@ impl<B: Backend> ForwardPassState<B> {
let size: usize = dims.iter().product();
let buffer_size = size * data_type.size_in_bytes();
let buffer = context.create_buffer(buffer_size).expect("Failed to create buffer");
- RefCell::new(unsafe { Array::from_parts(Rc::new(buffer), 0, dims, data_type) })
+ RefCell::new(unsafe { Array::from_parts(Rc::new(RefCell::new(buffer)), 0, dims, data_type) })
};
ClassifierModeState {
@@ -768,7 +777,7 @@ impl<B: Backend> ForwardPassState<B> {
pub fn encode_copy_array(
&self,
- command_buffer: &B::CommandBuffer,
+ command_buffer: &mut B::CommandBuffer,
source_array_id: ArrayId,
destination_array: RefCell<Array<B>>,
) {
@@ -776,14 +785,14 @@ impl<B: Backend> ForwardPassState<B> {
let src_borrow = source_ref.borrow();
let dst_borrow = destination_array.borrow();
- let src_buf = src_borrow.buffer();
- let dst_buf = dst_borrow.buffer();
+ let src_buf_rc = src_borrow.buffer();
+ let dst_buf_rc = dst_borrow.buffer();
let copy_size_bytes = dst_borrow.size();
debug_assert_eq!(dst_borrow.size(), src_borrow.size());
command_buffer.with_copy_encoder(|encoder| {
- encoder.encode_copy(src_buf, dst_buf, copy_size_bytes);
+ encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);
});
}
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 31450f93d..5cb0c89ea 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -1,4 +1,11 @@
-use std::{collections::HashMap, iter::repeat_n, ops::Deref, path::Path, sync::Arc, time::Instant};
+use std::{
+ collections::HashMap,
+ iter::repeat_n,
+ ops::{Deref, DerefMut},
+ path::Path,
+ sync::Arc,
+ time::Instant,
+};
use itertools::{Either, Itertools, izip};
@@ -402,7 +409,7 @@ where
let last_token = *self.tokens.last().ok_or(Error::PrefillFailed)?;
let token_position = unsafe {
- let ptr = async_positions_buffer.cpu_ptr().as_ptr() as *const u32;
+ let ptr = async_positions_buffer.borrow().cpu_ptr().as_ptr() as *const u32;
*ptr.add(pass_idx) as usize
};
@@ -468,30 +475,36 @@ where
self.context.executables.encode(
&mut state,
&EncodingParameters::new(false, false, false),
- self.context.command_buffer.borrow().deref(),
+ self.context.command_buffer.borrow_mut().deref_mut(),
);
// Encode sampling
self.context.gpu_sampler.encode(
&mut state,
&EncodingParameters::new(false, false, false),
- self.context.command_buffer.borrow().deref(),
+ self.context.command_buffer.borrow_mut().deref_mut(),
);
// Copy sampled token: sampling_output → token_ids (for next pass)
// and sampling_output → results[slot] (for callback)
let sampling_output = state.sampling_output().expect("sampling_output must exist after sampling encode");
let sampling_output_binding = sampling_output.borrow();
- let sampling_output_buffer = sampling_output_binding.buffer();
+ let sampling_output_buf_rc = sampling_output_binding.buffer();
+ let sampling_output_buf_borrow = sampling_output_buf_rc.borrow();
let token_ids_binding = self.context.scratch_buffers.token_ids.borrow();
- let token_ids_buffer = token_ids_binding.buffer();
+ let token_ids_buf_rc = token_ids_binding.buffer();
+ let token_ids_buf_borrow = token_ids_buf_rc.borrow();
- self.context.command_buffer.borrow().with_compute_encoder(|encoder| {
- self.context.token_copy_sampled.encode(sampling_output_buffer, token_ids_buffer, encoder);
+ self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {
+ self.context.token_copy_sampled.encode(
+ sampling_output_buf_borrow.deref(),
+ token_ids_buf_borrow.deref(),
+ encoder,
+ );
let results_offset = slot * std::mem::size_of::<u32>();
self.context.token_copy_results.encode(
- sampling_output_buffer,
- (results_buffer.as_ref(), results_offset),
+ sampling_output_buf_borrow.deref(),
+ (results_buffer.borrow().deref(), results_offset),
encoder,
);
});
@@ -500,24 +513,21 @@ where
self.context.cache_layers.borrow_mut().update_after_acceptance(
&[0],
None,
- self.context.command_buffer.borrow().deref(),
+ self.context.command_buffer.borrow_mut().deref_mut(),
&self.context.kv_cache_update,
);
self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
- self.context.command_buffer.borrow().with_compute_encoder(|encoder| {
+ self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {
if let Some(mask_update) = &self.context.mask_update {
let updates: Vec<AttentionBiasUpdate> =
self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
for (window_size, mask_buffer) in &self.context.scratch_buffers.attention_window_size_to_bias {
if let Some(update) = updates.iter().find(|u| &u.key == window_size) {
if update.unmask_col >= 0 || update.mask_col >= 0 {
- mask_update.encode(
- mask_buffer.borrow().buffer(),
- update.unmask_col,
- update.mask_col,
- &encoder,
- );
+ let mask_buf_rc = mask_buffer.borrow().buffer();
+ let mask_buf_borrow = mask_buf_rc.borrow();
+ mask_update.encode(mask_buf_borrow.deref(), update.unmask_col, update.mask_col, &encoder);
}
}
}
@@ -535,7 +545,7 @@ where
let handler = move || {
let token = {
- let ptr = results_buffer_clone.cpu_ptr().as_ptr() as *const u32;
+ let ptr = results_buffer_clone.borrow().cpu_ptr().as_ptr() as *const u32;
unsafe { *ptr.add(slot) as u64 }
};
if let Some(cb) = callback.lock().unwrap().take() {
@@ -652,7 +662,7 @@ where
self.context.gpu_sampler.encode(
&mut state,
&EncodingParameters::new(warmup, true, false),
- &self.context.command_buffer.as_ref().borrow(),
+ self.context.command_buffer.borrow_mut().deref_mut(),
);
}
}
@@ -713,14 +723,14 @@ where
suffix_start: Option<usize>,
wait_until_completed: bool,
) {
- let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");
{
let mut cache_layers = self.context.cache_layers.borrow_mut();
cache_layers.update_after_acceptance(
accepted_token_indices,
suffix_start,
- &command_buffer,
+ &mut command_buffer,
&self.context.kv_cache_update,
);
}
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index 1e97a1207..02912efbd 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -34,13 +34,13 @@ use crate::{
pub struct AsyncBuffers<B: Backend> {
/// Positions buffer: [max_tokens] i32
/// Pre-populated with [prefill_count, prefill_count+1, ...]
- pub positions: Rc<B::NativeBuffer>,
+ pub positions: Rc<RefCell<B::NativeBuffer>>,
/// Seeds buffer: [max_tokens] u64
/// Pre-populated with deterministic seed sequence
- pub seeds: Rc<B::NativeBuffer>,
+ pub seeds: Rc<RefCell<B::NativeBuffer>>,
/// Results buffer: [batch_size] u32
/// Each pass writes its sampled token to results[pass_idx % batch_size]
- pub results: Rc<B::NativeBuffer>,
+ pub results: Rc<RefCell<B::NativeBuffer>>,
/// Event for GPU-side synchronization between passes
pub event: B::Event,
/// Current event counter (pass N waits on N, signals N+1)
@@ -66,9 +66,9 @@ impl<B: Backend> AsyncBuffers<B> {
let event = context.create_event().expect("Failed to create event");
Self {
- positions: Rc::new(positions),
- seeds: Rc::new(seeds),
- results: Rc::new(results),
+ positions: Rc::new(RefCell::new(positions)),
+ seeds: Rc::new(RefCell::new(seeds)),
+ results: Rc::new(RefCell::new(results)),
event,
counter: Cell::new(0),
prefill_count: Cell::new(0),
@@ -85,7 +85,7 @@ impl<B: Backend> AsyncBuffers<B> {
) {
self.prefill_count.set(prefill_count);
let base_position = prefill_count.saturating_sub(1);
- let ptr = self.positions.cpu_ptr().as_ptr() as *mut i32;
+ let ptr = self.positions.borrow().cpu_ptr().as_ptr() as *mut i32;
for i in 0..tokens_to_generate {
unsafe {
*ptr.add(i) = (base_position + i) as i32;
@@ -100,7 +100,7 @@ impl<B: Backend> AsyncBuffers<B> {
prefix_len: usize,
tokens_to_generate: usize,
) {
- let ptr = self.seeds.cpu_ptr().as_ptr() as *mut u64;
+ let ptr = self.seeds.borrow().cpu_ptr().as_ptr() as *mut u64;
for i in 0..tokens_to_generate {
unsafe {
*ptr.add(i) = seed.derive((prefix_len + i - 1) as u64);
@@ -118,7 +118,7 @@ impl<B: Backend> AsyncBuffers<B> {
&self,
pass_idx: usize,
) -> u32 {
- let ptr = self.results.cpu_ptr().as_ptr() as *const u32;
+ let ptr = self.results.borrow().cpu_ptr().as_ptr() as *const u32;
unsafe { *ptr.add(pass_idx % self.batch_size) }
}
}
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index e34b9ac29..fc419fa14 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -1,4 +1,4 @@
-use std::{mem::size_of, ops::Deref};
+use std::{mem::size_of, ops::DerefMut};
use super::LanguageModelGeneratorContext;
use crate::{
@@ -17,7 +17,7 @@ impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {
&self.predicate_buffer
}
- pub fn disable_execution(&self) {
+ pub fn disable_execution(&mut self) {
let ptr = self.predicate_buffer.cpu_ptr().as_ptr() as *mut u32;
unsafe {
@@ -114,7 +114,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
parameters: &EncodingParameters<B>,
key: String,
) -> LanguageModelGeneratorEncodedTask<B> {
- context.executables.encode(state, parameters, context.command_buffer.borrow().deref());
+ context.executables.encode(state, parameters, context.command_buffer.borrow_mut().deref_mut());
LanguageModelGeneratorEncodedTask {
key,
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 18a6d4897..c3e3f5d43 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -282,9 +282,9 @@ where
None,
);
- let command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");
+ let mut command_buffer = ctx.context.create_command_buffer().expect("Failed to create command buffer");
- ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &command_buffer);
+ ctx.executables.encode(&mut state, &EncodingParameters::new(false, false, false), &mut command_buffer);
command_buffer.submit();
command_buffer.wait_until_completed();
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index 488953452..4573536af 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -1,6 +1,7 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
use metal::{MTLCommandBuffer, MTLCommandQueue};
+use objc2::Message;
use uzu::{
DataType,
array::ArrayContextExt,
@@ -190,7 +191,7 @@ fn run_scenario(
layer.update_after_acceptance(
&scenario.accepted_suffix_indices,
scenario.suffix_start,
- &root_command_buffer,
+ &mut root_command_buffer.retain(),
&kv_cache_update,
);
diff --git a/crates/uzu/tests/kv_cache_update_test.rs b/crates/uzu/tests/kv_cache_update_test.rs
index 6ce00197d..90b93f9d6 100644
--- a/crates/uzu/tests/kv_cache_update_test.rs
+++ b/crates/uzu/tests/kv_cache_update_test.rs
@@ -1,6 +1,6 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
-use std::rc::Rc;
+use std::{cell::RefCell, rc::Rc};
use bytemuck;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
@@ -81,23 +81,23 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
let device = &context.device;
- let key_buffer = Rc::new(
+ let key_buffer = Rc::new(RefCell::new(
device
.new_buffer_with_data(
bytemuck::cast_slice(key_data.as_slice().unwrap()),
MTLResourceOptions::STORAGE_MODE_SHARED,
)
.expect("Failed to create buffer"),
- );
+ ));
- let value_buffer = Rc::new(
+ let value_buffer = Rc::new(RefCell::new(
device
.new_buffer_with_data(
bytemuck::cast_slice(value_data.as_slice().unwrap()),
MTLResourceOptions::STORAGE_MODE_SHARED,
)
.expect("Failed to create buffer"),
- );
+ ));
let kv_layer_data = KVLayerData::<Metal> {
key_buffer: key_buffer.clone(),
@@ -106,8 +106,9 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
value_shape: [num_heads, seq_len, head_dim],
};
- let command_buffer = context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &command_buffer) {
+ let mut command_buffer =
+ context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
+ match kv_cache_update.encode(&[kv_layer_data], &source_indices, &destination_indices, &mut command_buffer) {
Ok(_) => {},
Err(e) => {
println!("Warning: Failed to encode KV cache update: {:?}. Skipping test.", e);
@@ -118,8 +119,8 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
command_buffer.commit();
command_buffer.wait_until_completed();
- let key_result_ptr = key_buffer.contents().as_ptr() as *const f32;
- let value_result_ptr = value_buffer.contents().as_ptr() as *const f32;
+ let key_result_ptr = key_buffer.borrow().contents().as_ptr() as *const f32;
+ let value_result_ptr = value_buffer.borrow().contents().as_ptr() as *const f32;
let total_elems = num_heads * seq_len * head_dim;
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index b6a53be53..874a4b28b 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -346,7 +346,7 @@ fn run_moe_parity_test_internal(
// Encode ALL kernels in one command buffer
eprintln!("[E2E] Encoding entire MoE pipeline in single command buffer...");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
// Router + TopK (fused kernel)
let router_topk =
@@ -412,7 +412,7 @@ fn run_moe_parity_test_internal(
let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("gather");
gather.encode(
- &cb,
+ &mut cb,
DataType::BF16,
&MoeGatherArguments {
x_buffer: &x_buf,
@@ -427,16 +427,16 @@ fn run_moe_parity_test_internal(
// Additional buffers for 2-pass
let total_rows = t * k;
- let hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, total_rows * d_ff);
let row_expert_map_buf = alloc_buffer::<u32>(&ctx, total_rows);
let experts = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("experts");
let num_tiles_k = ((d_ff + 64 - 1) / 64) as u32;
- let args = MoeExpertsTwoPassArguments {
+ let mut args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
+ hidden_buffer: &mut hidden_buf,
output_buffer: &y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
@@ -460,7 +460,7 @@ fn run_moe_parity_test_internal(
silu_alpha,
data_type: DataType::BF16,
};
- experts.encode(&cb, &args);
+ experts.encode(&mut cb, &mut args);
let finalize =
<<Metal as Backend>::Kernels as Kernels>::MoeFinalizeKernel::new(&ctx, DataType::BF16).expect("finalize");
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index 07f20f91b..a71375bc2 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -93,7 +93,7 @@ fn run_decode_case(
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;
- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
// Buffers for indirect dispatch
@@ -105,38 +105,39 @@ fn run_decode_case(
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);
- let make_two_pass_args = || MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &output_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- data_type: DataType::BF16,
- };
-
for _ in 0..warmup {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, &make_two_pass_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ experts_kernel.encode(
+ &mut cb,
+ &MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &mut hidden_buf,
+ output_buffer: &output_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.702,
+ data_type: DataType::BF16,
+ },
+ );
cb.commit();
cb.wait_until_completed();
}
@@ -144,8 +145,38 @@ fn run_decode_case(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, &make_two_pass_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ experts_kernel.encode(
+ &mut cb,
+ &MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &mut hidden_buf,
+ output_buffer: &output_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.702,
+ data_type: DataType::BF16,
+ },
+ );
cb.commit();
cb.wait_until_completed();
times.push(start.elapsed().as_secs_f64() * 1000.0);
@@ -227,7 +258,7 @@ fn run_two_pass_prefill_case(
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;
- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
let output_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
@@ -238,38 +269,37 @@ fn run_two_pass_prefill_case(
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);
- let make_two_pass_args = || MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &output_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.702,
- data_type: DataType::BF16,
- };
-
for _ in 0..warmup {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, &make_two_pass_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut args = MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &mut hidden_buf,
+ output_buffer: &output_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.702,
+ data_type: DataType::BF16,
+ };
+ experts_kernel.encode(&mut cb, &mut args);
cb.commit();
cb.wait_until_completed();
}
@@ -277,8 +307,36 @@ fn run_two_pass_prefill_case(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, &make_two_pass_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut args = MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &mut hidden_buf,
+ output_buffer: &output_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.702,
+ data_type: DataType::BF16,
+ };
+ experts_kernel.encode(&mut cb, &mut args);
cb.commit();
cb.wait_until_completed();
times.push(start.elapsed().as_secs_f64() * 1000.0);
@@ -377,8 +435,8 @@ fn run_fused_single_token_case(
};
for _ in 0..warmup {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- fused_kernel.encode(&cb, make_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ fused_kernel.encode(&mut cb, make_args());
cb.commit();
cb.wait_until_completed();
}
@@ -386,8 +444,8 @@ fn run_fused_single_token_case(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- fused_kernel.encode(&cb, make_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ fused_kernel.encode(&mut cb, make_args());
cb.commit();
cb.wait_until_completed();
times.push(start.elapsed().as_secs_f64() * 1000.0);
@@ -499,7 +557,7 @@ fn run_indirect_decode_timed(
let down_biases_buf = alloc_buffer_with_data(ctx, &down_biases);
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as usize;
- let hidden_buf = alloc_buffer::<f32>(ctx, sum_k * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(ctx, sum_k * d_ff);
let output_buf = alloc_buffer::<bf16>(ctx, sum_k * d_model);
let tile_counts_buf = alloc_buffer::<u32>(ctx, e);
@@ -510,38 +568,39 @@ fn run_indirect_decode_timed(
let dispatch_args_buf = alloc_buffer::<u32>(ctx, 3);
let row_expert_map_buf = alloc_buffer::<u32>(ctx, sum_k);
- let make_args = || MoeExpertsTwoPassArguments {
- x_perm_buffer: &x_perm_buf,
- expert_offsets: &offsets_buf,
- row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
- output_buffer: &output_buf,
- w13_all: &w13_buf,
- w2_all: &w2_buf,
- up_biases: &up_biases_buf,
- down_biases: &down_biases_buf,
- tile_counts: &tile_counts_buf,
- tile_offsets: &tile_offsets_buf,
- tile_map: &tile_map_buf,
- total_tiles: &total_tiles_buf,
- dispatch_args: &dispatch_args_buf,
- total_rows: sum_k,
- d_model,
- d_ff,
- e,
- num_tiles_k: num_tiles_k as u32,
- gating_code: 2,
- gate_clip_min: f32::NEG_INFINITY,
- gate_clip_max: f32::INFINITY,
- up_clip_min: f32::NEG_INFINITY,
- up_clip_max: f32::INFINITY,
- silu_alpha: 1.0,
- data_type: DataType::BF16,
- };
-
for _ in 0..warmup {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, &make_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ experts_kernel.encode(
+ &mut cb,
+ &MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &mut hidden_buf,
+ output_buffer: &output_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.0,
+ data_type: DataType::BF16,
+ },
+ );
cb.commit();
cb.wait_until_completed();
}
@@ -549,8 +608,38 @@ fn run_indirect_decode_timed(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- experts_kernel.encode(&cb, &make_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ experts_kernel.encode(
+ &mut cb,
+ &MoeExpertsTwoPassArguments {
+ x_perm_buffer: &x_perm_buf,
+ expert_offsets: &offsets_buf,
+ row_expert_map: &row_expert_map_buf,
+ hidden_buffer: &mut hidden_buf,
+ output_buffer: &output_buf,
+ w13_all: &w13_buf,
+ w2_all: &w2_buf,
+ up_biases: &up_biases_buf,
+ down_biases: &down_biases_buf,
+ tile_counts: &tile_counts_buf,
+ tile_offsets: &tile_offsets_buf,
+ tile_map: &tile_map_buf,
+ total_tiles: &total_tiles_buf,
+ dispatch_args: &dispatch_args_buf,
+ total_rows: sum_k,
+ d_model,
+ d_ff,
+ e,
+ num_tiles_k: num_tiles_k as u32,
+ gating_code: 2,
+ gate_clip_min: f32::NEG_INFINITY,
+ gate_clip_max: f32::INFINITY,
+ up_clip_min: f32::NEG_INFINITY,
+ up_clip_max: f32::INFINITY,
+ silu_alpha: 1.0,
+ data_type: DataType::BF16,
+ },
+ );
cb.commit();
cb.wait_until_completed();
times.push(start.elapsed().as_secs_f64() * 1000.0);
@@ -621,8 +710,8 @@ fn run_fused_decode_timed(
};
for _ in 0..warmup {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- fused_kernel.encode(&cb, make_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ fused_kernel.encode(&mut cb, make_args());
cb.commit();
cb.wait_until_completed();
}
@@ -630,8 +719,8 @@ fn run_fused_decode_timed(
let mut times = Vec::with_capacity(iters);
for _ in 0..iters {
let start = Instant::now();
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- fused_kernel.encode(&cb, make_args());
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ fused_kernel.encode(&mut cb, make_args());
cb.commit();
cb.wait_until_completed();
times.push(start.elapsed().as_secs_f64() * 1000.0);
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index 8bcd8bc53..3222ef730 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -401,7 +401,7 @@ fn test_two_pass_decode_correctness() {
let up_biases_buf = alloc_buffer_with_data(&ctx, &up_biases);
let down_biases_buf = alloc_buffer_with_data(&ctx, &down_biases);
- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
// Tile infrastructure
@@ -416,18 +416,18 @@ fn test_two_pass_decode_correctness() {
// Execute 2-pass decode kernel
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("MoeExpertsTwoPassDecodeKernel::new");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
const K_TILE: usize = 64;
let num_tiles_k = ((d_ff + K_TILE - 1) / K_TILE) as u32;
experts_kernel.encode(
- &cb,
+ &mut cb,
&MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
+ hidden_buffer: &mut hidden_buf,
output_buffer: &y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
@@ -548,7 +548,7 @@ fn test_two_pass_decode_multi_token() {
let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);
- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
// Tile infrastructure
@@ -560,14 +560,14 @@ fn test_two_pass_decode_multi_token() {
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassDecodeBlock::<Metal>::new(&ctx).expect("kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(
- &cb,
+ &mut cb,
&MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
+ hidden_buffer: &mut hidden_buf,
output_buffer: &y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
@@ -650,7 +650,7 @@ fn test_two_pass_prefill_correctness() {
let w2_buf = alloc_buffer_with_data(&ctx, &data.w2);
let up_biases_buf = alloc_buffer_with_data(&ctx, &data.up_biases);
let down_biases_buf = alloc_buffer_with_data(&ctx, &data.down_biases);
- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
let y_partial_buf = alloc_buffer::<bf16>(&ctx, sum_k * d_model);
// Tile infrastructure
@@ -662,12 +662,12 @@ fn test_two_pass_prefill_correctness() {
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
let experts_kernel = MoeExpertsTwoPassPrefillBlock::<Metal>::new(&ctx).expect("kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
- let args = MoeExpertsTwoPassArguments {
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut args = MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
+ hidden_buffer: &mut hidden_buf,
output_buffer: &y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
@@ -691,7 +691,7 @@ fn test_two_pass_prefill_correctness() {
silu_alpha,
data_type: DataType::BF16,
};
- experts_kernel.encode(&cb, &args);
+ experts_kernel.encode(&mut cb, &mut args);
cb.commit();
cb.wait_until_completed();
@@ -791,10 +791,10 @@ fn test_fused_single_token_decode() {
// Run fused decode kernel
let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("MoeExpertsSingleDecodeKernel::new");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(
- &cb,
+ &mut cb,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
@@ -915,9 +915,9 @@ fn test_fused_single_token_k4() {
let y_buf = alloc_buffer::<bf16>(&ctx, d_model);
let fused_kernel = MoeExpertsSingleDecodeKernels::<Metal>::new(&ctx).expect("fused kernel");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
fused_kernel.encode(
- &cb,
+ &mut cb,
MoeExpertsSingleDecodeArguments {
x: &x_buf,
topk_ids: &topk_ids_buf,
diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rs
index 928823bd9..62ef950a5 100644
--- a/crates/uzu/tests/moe/moe_gather_test.rs
+++ b/crates/uzu/tests/moe/moe_gather_test.rs
@@ -78,9 +78,9 @@ fn test_gather_correctness() {
// Execute gather kernel using kernel struct
let gather = MoeGatherKernels::<Metal>::new(&ctx).expect("MoeGatherKernel::new");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
gather.encode(
- &cb,
+ &mut cb,
DataType::BF16,
&MoeGatherArguments {
x_buffer: &x_buf,
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index a81c6af36..f62468bb1 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -285,7 +285,7 @@ fn test_moe_pipeline_breakdown_decode() {
let dispatch_args_buf = alloc_buffer::<u32>(&ctx, 3);
// Two-pass specific buffers
- let hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
+ let mut hidden_buf = alloc_buffer::<f32>(&ctx, sum_k * d_ff);
let row_expert_map_buf = alloc_buffer::<u32>(&ctx, sum_k);
// Scatter block bases buffers
@@ -388,9 +388,9 @@ fn test_moe_pipeline_breakdown_decode() {
});
let gather_perf = time_kernel("Gather", 2, 5, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
gather_kernel.encode(
- &cb,
+ &mut cb,
DataType::BF16,
&MoeGatherArguments {
x_buffer: &x_buf,
@@ -407,14 +407,14 @@ fn test_moe_pipeline_breakdown_decode() {
});
let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
experts_kernel.encode(
- &cb,
+ &mut cb,
&MoeExpertsTwoPassArguments {
x_perm_buffer: &x_perm_buf,
expert_offsets: &offsets_buf,
row_expert_map: &row_expert_map_buf,
- hidden_buffer: &hidden_buf,
+ hidden_buffer: &mut hidden_buf,
output_buffer: &y_partial_buf,
w13_all: &w13_buf,
w2_all: &w2_buf,
diff --git a/crates/uzu/tests/moe/moe_tiles_test.rs b/crates/uzu/tests/moe/moe_tiles_test.rs
index 874d41329..d72ddfd02 100644
--- a/crates/uzu/tests/moe/moe_tiles_test.rs
+++ b/crates/uzu/tests/moe/moe_tiles_test.rs
@@ -35,9 +35,9 @@ fn test_tile_counts_correctness() {
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
- &cb,
+ &mut cb,
&MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &tile_counts_buf,
@@ -80,9 +80,9 @@ fn test_tile_scan_correctness() {
// Execute kernel using kernel struct
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_scan(
- &cb,
+ &mut cb,
&MoeTileScanArguments {
tile_counts_buffer: &tile_counts_buf,
tile_offsets_buffer: &tile_offsets_buf,
@@ -120,9 +120,9 @@ fn test_tile_edge_cases() {
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
- &cb,
+ &mut cb,
&MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &tile_counts_buf,
@@ -151,9 +151,9 @@ fn test_tile_edge_cases() {
let tile_counts_buf = alloc_buffer::<u32>(&ctx, e);
let tile_kernel = MoeTileMapKernels::<Metal>::new(&ctx).expect("MoeTileMapKernel::new");
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
+ let mut cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
tile_kernel.encode_counts(
- &cb,
+ &mut cb,
&MoeTileCountsArguments {
offsets_buffer: &offsets_buf,
tile_counts_buffer: &tile_counts_buf,
diff --git a/crates/uzu/tests/person_schema_metal_bitmask_test.rs b/crates/uzu/tests/person_schema_metal_bitmask_test.rs
index 1b181ffe7..4e4e1fcc6 100644
--- a/crates/uzu/tests/person_schema_metal_bitmask_test.rs
+++ b/crates/uzu/tests/person_schema_metal_bitmask_test.rs
@@ -1,6 +1,6 @@
#![cfg(target_os = "macos")]
-use std::{path::PathBuf, rc::Rc};
+use std::{cell::RefCell, path::PathBuf, rc::Rc};
use metal::{MTLDevice, MTLDeviceExt, MTLResourceOptions};
use schemars::JsonSchema;
@@ -62,7 +62,7 @@ fn person_schema_metal_bitmask() {
let elems = batch * buffer_size;
let bytes = elems * core::mem::size_of::<i32>();
let buffer = device.new_buffer(bytes, MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");
- let metal_bitmask = unsafe { Array::<Metal>::from_parts(Rc::new(buffer), 0, &shape, DataType::I32) };
+ let metal_bitmask = unsafe { Array::<Metal>::from_parts(Rc::new(RefCell::new(buffer)), 0, &shape, DataType::I32) };
let mut shape_i64 = [buffer_size as i64];
let mut bitmask_tensor = DLTensor {