A high-performance inference engine for AI models
array getter cleanup (#291)
uuuvn4cfcad47172b
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex b89f34fb7..42e39e5ca 100644--- a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -353,7 +353,7 @@ impl StructuredAudioCodecGraph {1,);- let main = state.arrays(&[ArrayId::Main])[0].clone();+ let main = state.array(ArrayId::Main);let main_output = {if main.shape() != [frames, self.input_dim] {return Err(AudioError::Runtime(format!(@@ -462,7 +462,7 @@ impl StructuredAudioCodecGraph {1,);- let main = state.arrays(&[ArrayId::Main])[0].clone();+ let main = state.array(ArrayId::Main);let main_output = {if main.shape() != [active_len, self.input_dim] {return Err(AudioError::Runtime(format!(diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rsindex 491431e61..b55c6214d 100644--- a/crates/uzu/src/backends/common/kernel/sampling.rs+++ b/crates/uzu/src/backends/common/kernel/sampling.rs@@ -58,8 +58,6 @@ pub enum SamplingError<B: Backend> {BatchSizeExceeded(usize, usize),#[error("Vocab size {0} exceeds maximum {1}")]VocabSizeExceeded(usize, usize),- #[error("Stochastic sampling encode must have a seed")]- StochasticWithoutSeed,}impl<B: Backend> SamplingKernel<B> {@@ -144,7 +142,7 @@ impl<B: Backend> SamplingKernel<B> {pub fn encode(&self,mut logits_buffer: &mut B::Buffer,- seeds_buffer: Option<&B::Buffer>,+ seeds_buffer: &B::Buffer,seeds_offset: usize,bitmask_buffer: Option<&B::Buffer>,bitmask_offset: usize,@@ -239,7 +237,7 @@ impl<B: Backend> SamplingKernel<B> {self.gumbel.encode(None::<&B::Buffer>,- (seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),+ (seeds_buffer, seeds_offset),logits_buffer.deref_mut(),batch_size as u32,vocab_size as u32,diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 6b9f0bd02..f8c34c890 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -186,25 +186,13 @@ impl<B: Backend> Classifier<B> {&self,state: &ForwardPassState<B>,) -> Result<Box<[f32]>, Error> {- let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);- let logits_array = &logits_arrays[0];-let num_labels = self.context.model_config.model_config.num_labels;- let buffer = logits_array.as_bytes();-- match logits_array.data_type() {- DataType::F32 => {- let slice: &[f32] = bytemuck::cast_slice(buffer);- Ok(slice[..num_labels].into())- },- DataType::F16 => {- let slice: &[half::f16] = bytemuck::cast_slice(buffer);- Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())- },- DataType::BF16 => {- let slice: &[half::bf16] = bytemuck::cast_slice(buffer);- Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())- },+ let logits = state.array(ArrayId::ClassifierPredictionHeadLogits).view(&[num_labels]);++ match logits.data_type() {+ DataType::F32 => Ok(logits.as_slice::<f32>().into()),+ DataType::F16 => Ok(logits.as_slice::<half::f16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),+ DataType::BF16 => Ok(logits.as_slice::<half::bf16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),_ => Err(Error::UnableToDecodeText),}}diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rsindex 27177bea0..55598deec 100644--- a/crates/uzu/src/encodable_block/activation.rs+++ b/crates/uzu/src/encodable_block/activation.rs@@ -42,9 +42,8 @@ impl<B: Backend> Activation<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);- let input_array = &arrays[0];- let output_array = &arrays[1];+ let input_array = state.array(self.input_array_id);+ let output_array = state.array(self.output_array_id);let n = input_array.shape().iter().product::<usize>();if self.activation.act_type() == ActivationType::IDENTITY {diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex 2c33295d5..87f65dfb0 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -149,30 +149,23 @@ impl<B: Backend> Attention<B> {parameters: &EncodingParameters,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {- let qkv_binding = state.arrays(&[ArrayId::QKV]);- let qkv_array = &qkv_binding[0];- let suffix_length = qkv_array.shape()[0];-- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);- let queries_array = &queries_binding[0];- let num_heads = queries_array.shape()[0];- let head_dim = queries_array.shape()[2];-- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let keys_array = &keys_binding[0];- let num_groups = keys_array.shape()[0];-- let max_sequence_length = if let Some(_kv) = state.cache_layers() {- let key_cache_binding = state.arrays(&[ArrayId::Keys(self.layer_index)]);- let key_cache_array = &key_cache_binding[0];- key_cache_array.shape()[1]- } else {- // For classifiers without KV cache, max_sequence_length is just suffix_length- suffix_length- };-- (suffix_length, num_heads, head_dim, num_groups, max_sequence_length)+ let qkv_array = state.array(ArrayId::QKV);+ let queries_array = state.array(ArrayId::RotatedQueries);+ let rotated_keys_array = state.array(ArrayId::RotatedKeys);+ let attention_output_array = state.array(ArrayId::AttentionOutput);+ let partials_array = state.array(ArrayId::AttentionPartials);+ let sums_array = state.array(ArrayId::AttentionSums);+ let maxs_array = state.array(ArrayId::AttentionMaxs);++ let suffix_length = qkv_array.shape()[0];+ let num_heads = queries_array.shape()[0];+ let head_dim = queries_array.shape()[2];+ let num_groups = rotated_keys_array.shape()[0];+ let max_sequence_length = if state.cache_layers().is_some() {+ state.array(ArrayId::Keys(self.layer_index)).shape()[1]+ } else {+ // For classifiers without KV cache, max_sequence_length is just suffix_length+ suffix_length};let (segment_prefix_length, window_length) = if let Some(cache_layers) = state.cache_layers() {@@ -202,45 +195,32 @@ impl<B: Backend> Attention<B> {}let variant = self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, use_mask);- let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let qkv_binding = state.arrays(&[ArrayId::QKV]);let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);- let attention_output_binding = state.arrays(&[ArrayId::AttentionOutput]);- let gate_binding = self.gate_kernel.as_ref().map(|_| state.arrays(&[ArrayId::Gate]));+ let gate_array = self.gate_kernel.as_ref().map(|_| state.array(ArrayId::Gate));let mask_kv_seq_stride = 1;let mask_q_seq_stride = sequence_length as i32;let mask_head_stride = 0;- let sinks_binding = if self.has_sinks {- Some(state.arrays(&[ArrayId::AttentionSinks(self.layer_index)]))- } else {- None- };+ let sinks_array = self.has_sinks.then(|| state.array(ArrayId::AttentionSinks(self.layer_index)));- let rotated_keys_array = &rotated_keys_binding[0];let rotated_keys_buf_rc = rotated_keys_array.buffer();let mut rotated_keys_buf_borrow = rotated_keys_buf_rc.borrow_mut();- let qkv_array = &qkv_binding[0];let qkv_buf_rc = qkv_array.buffer();let qkv_buf_borrow = qkv_buf_rc.borrow();// Get KV cache buffers only if KV cache exists (LLM mode)let has_kv_cache = state.cache_layers().is_some();- let key_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Keys(self.layer_index)]));- let key_cache_array = key_cache_binding.as_ref().map(|b| &b[0]);+ let key_cache_array = has_kv_cache.then(|| state.array(ArrayId::Keys(self.layer_index)));let key_cache_buf_rc = key_cache_array.as_ref().map(|a| a.buffer());let mut key_cache_buf_borrow = key_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());- let value_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Values(self.layer_index)]));- let value_cache_array = value_cache_binding.as_ref().map(|b| &b[0]);+ let value_cache_array = has_kv_cache.then(|| state.array(ArrayId::Values(self.layer_index)));let value_cache_buf_rc = value_cache_array.as_ref().map(|a| a.buffer());let mut value_cache_buf_borrow = value_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());- let extracted_values_binding = (!has_kv_cache).then(|| state.arrays(&[ArrayId::ExtractedValues]));- let extracted_values_array = extracted_values_binding.as_ref().map(|b| &b[0]);+ let extracted_values_array = (!has_kv_cache).then(|| state.array(ArrayId::ExtractedValues));let extracted_values_buf_rc = extracted_values_array.as_ref().map(|a| a.buffer());let mut extracted_values_buf_borrow = extracted_values_buf_rc.as_ref().map(|rc| rc.borrow_mut());@@ -262,11 +242,9 @@ impl<B: Backend> Attention<B> {);}- let queries_array = &rotated_queries_binding[0];let queries_buf_rc = queries_array.buffer();let queries_buf_borrow = queries_buf_rc.borrow();- let attention_output_array = &attention_output_binding[0];let attention_output_buf_rc = attention_output_array.buffer();let mut attention_output_buf_borrow = attention_output_buf_rc.borrow_mut();@@ -283,24 +261,16 @@ impl<B: Backend> Attention<B> {let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());let attention_bias_buffer: Option<&B::Buffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());- let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);- let sums_binding = state.arrays(&[ArrayId::AttentionSums]);- let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);-- let partials_array = &partials_binding[0];let partials_buf_rc = partials_array.buffer();let mut partials_buf_borrow = partials_buf_rc.borrow_mut();- let sums_array = &sums_binding[0];let sums_buf_rc = sums_array.buffer();let mut sums_buf_borrow = sums_buf_rc.borrow_mut();- let maxs_array = &maxs_binding[0];let maxs_buf_rc = maxs_array.buffer();let mut maxs_buf_borrow = maxs_buf_rc.borrow_mut();- let sinks_borrow = sinks_binding.as_ref().map(|binding| &binding[0]);- let sinks_buf_rc = sinks_borrow.as_ref().map(|b| b.buffer());+ let sinks_buf_rc = sinks_array.as_ref().map(|b| b.buffer());let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());let sinks_buffer: Option<&B::Buffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());@@ -444,7 +414,7 @@ impl<B: Backend> Attention<B> {}if let Some(gate_kernel) = &self.gate_kernel {- let gate_array = &gate_binding.as_ref().unwrap()[0];+ let gate_array = gate_array.as_ref().unwrap();let gate_buf_rc = gate_array.buffer();let gate_buf_borrow = gate_buf_rc.borrow();let total_elements = (suffix_length * num_heads * head_dim) as u32;diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex a2344e829..313a603e6 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -48,12 +48,8 @@ impl<B: Backend> ClassifierLayer<B> {let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();- let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap();+ let copy_main_to_shortcut_mixer =+ TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {if layer_loader.subtree("pre_mixer_norm").is_ok() {@@ -134,19 +130,11 @@ impl<B: Backend> ClassifierLayer<B> {None};- let mixer_residual_add = TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap();+ let mixer_residual_add =+ TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();- let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap();+ let copy_main_to_shortcut_mlp =+ TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();let pre_mlp_norm = Normalization::new(ctx,@@ -195,12 +183,8 @@ impl<B: Backend> ClassifierLayer<B> {).expect("Failed to create attention kernel");- let mlp_residual_add = TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap();+ let mlp_residual_add =+ TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();Self {layer_index,diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex 0ea8075ed..4c21fc68c 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -493,14 +493,12 @@ impl<B: Backend> Embedding<B> {) -> Result<(), EmbeddingError<B>> {let batch_size = state.active_suffix_length() as u32;- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);-- let token_ids_array = &arrays[0];+ let token_ids_array = state.array(ArrayId::TokenIds);let token_ids_buffer_rc = token_ids_array.buffer();let token_ids_buffer_borrow = token_ids_buffer_rc.borrow();let token_ids = token_ids_buffer_borrow.deref();- let output_array = &arrays[1];+ let output_array = state.array(ArrayId::Main);let output_buffer_rc = output_array.buffer();let mut output_buffer_borrow = output_buffer_rc.borrow_mut();let output = output_buffer_borrow.deref_mut();@@ -580,14 +578,12 @@ impl<B: Backend> Embedding<B> {return Ok(());}- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);-- let input_array = &arrays[0];+ let input_array = state.array(ArrayId::Main);let input_buffer_rc = input_array.buffer();let input_buffer_borrow = input_buffer_rc.borrow();let input = input_buffer_borrow.deref();- let output_array = &arrays[1];+ let output_array = state.array(ArrayId::Logits);let output_buffer_rc = output_array.buffer();let mut output_buffer_borrow = output_buffer_rc.borrow_mut();let output = output_buffer_borrow.deref_mut();diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 2fb5461b3..641375307 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -51,12 +51,8 @@ impl<B: Backend> LayerExecutables<B> {MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),MixerConfig::DeltaNet(config) => config.in_proj_config.activation_precision().into(),};- let copy_main_to_shortcut = TensorCopy::<B>::new(- context,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap();+ let copy_main_to_shortcut =+ TensorCopy::<B>::new(context, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();let pre_attention_norm = RMSNorm::new(context,@@ -203,12 +199,8 @@ impl<B: Backend> LayerExecutables<B> {None};- let main_shortcut_add_swap = TensorAddSwap::<B>::new(- context,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap();+ let main_shortcut_add_swap =+ TensorAddSwap::<B>::new(context, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();let pre_mlp_norm = RMSNorm::new(context,diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rsindex 8f6deec5c..e811bdfb5 100644--- a/crates/uzu/src/encodable_block/layer_norm.rs+++ b/crates/uzu/src/encodable_block/layer_norm.rs@@ -73,19 +73,11 @@ impl<B: Backend> LayerNorm<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let input_binding = state.arrays(&[self.input_array_id]);- let output_binding = state.arrays(&[self.output_array_id]);+ let input_array = state.array(self.input_array_id);+ let output_array = state.array(self.output_array_id);- let input_shape = {- let input_array = &input_binding[0];- input_array.shape().to_vec()- };-- let input_array = &input_binding[0];- let output_array = &output_binding[0];-- let batch_size = input_shape[0] as u32;- let model_dim = input_shape[1] as u32;+ let batch_size = input_array.shape()[0] as u32;+ let model_dim = input_array.shape()[1] as u32;let full_layer = if self.config.upcast_mode == UpcastMode::FullLayer {1u32} else {@@ -94,6 +86,7 @@ impl<B: Backend> LayerNorm<B> {let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());+self.kernel.encode(input_buffer_borrow.as_deref(),self.scales_buffer.borrow().deref(),diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rsindex 686baffdc..64000ebba 100644--- a/crates/uzu/src/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs@@ -131,10 +131,9 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();- let input_array = &arrays[0];- let output_array = &arrays[1];+ let input_array = state.array(self.input_array_id);+ let output_array = state.array(self.output_array_id);let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());self.kernel.borrow_mut().encode(diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rsindex f88d95ef5..360617310 100644--- a/crates/uzu/src/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/encodable_block/linear/quantized.rs@@ -251,10 +251,10 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();- let input_array = &arrays[0];- let output_array = &arrays[1];++ let input_array = state.array(self.input_array_id);+ let output_array = state.array(self.output_array_id);let output_buf_rc = output_array.buffer();let mut output_buf_borrow = output_buf_rc.borrow_mut();diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex 4bd31cca3..f7a5862b1 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -149,20 +149,11 @@ impl<B: Backend> MambaMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) {- let arrays = state.arrays(&[- ArrayId::SsmInProj,- ArrayId::SsmPacked(self.layer_index),- ArrayId::SsmZ(self.layer_index),- ArrayId::SsmDt(self.layer_index),- ]);-- let in_proj = &arrays[0];- let conv_inputs = &arrays[1];- let gate = &arrays[2];- let dt = &arrays[3];+ let in_proj = state.array(ArrayId::SsmInProj);+ let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));+ let gate = state.array(ArrayId::SsmZ(self.layer_index));+ let dt = state.array(ArrayId::SsmDt(self.layer_index));- let bias_buf_rc = self.gate_bias.buffer();- let bias_buf_borrow = bias_buf_rc.borrow();let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let num_heads = self.config.num_heads;@@ -173,7 +164,7 @@ impl<B: Backend> MambaMixer<B> {conv_inputs.buffer().borrow_mut().deref_mut(),gate.buffer().borrow_mut().deref_mut(),dt.buffer().borrow_mut().deref_mut(),- bias_buf_borrow.deref(),+ self.gate_bias.buffer().borrow().deref(),suffix_length as u32,total_dim as u32,conv_dim as u32,@@ -189,18 +180,11 @@ impl<B: Backend> MambaMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) {- let arrays = state.arrays(&[- ArrayId::SsmPacked(self.layer_index),- ArrayId::SsmConvState(self.layer_index),- ArrayId::SsmX(self.layer_index),- ArrayId::SsmB(self.layer_index),- ArrayId::SsmC(self.layer_index),- ]);- let conv_inputs = &arrays[0];- let conv_state = &arrays[1];- let x_arr = &arrays[2];- let b_arr = &arrays[3];- let c_arr = &arrays[4];+ let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));+ let conv_state = state.array(ArrayId::SsmConvState(self.layer_index));+ let x_arr = state.array(ArrayId::SsmX(self.layer_index));+ let b_arr = state.array(ArrayId::SsmB(self.layer_index));+ let c_arr = state.array(ArrayId::SsmC(self.layer_index));let weight_buf_rc = self.conv_weight.buffer();let weight_buf_borrow = weight_buf_rc.borrow();@@ -288,26 +272,14 @@ impl<B: Backend> MambaMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) {- let base_arrays = state.arrays(&[- ArrayId::SsmX(self.layer_index),- ArrayId::SsmB(self.layer_index),- ArrayId::SsmC(self.layer_index),- ArrayId::SsmDt(self.layer_index),- ArrayId::SsmZ(self.layer_index),- ArrayId::SsmState(self.layer_index),- ArrayId::AttentionOutput,- ]);+ let x = state.array(ArrayId::SsmX(self.layer_index));+ let b = state.array(ArrayId::SsmB(self.layer_index));+ let c = state.array(ArrayId::SsmC(self.layer_index));+ let dt = state.array(ArrayId::SsmDt(self.layer_index));+ let z = state.array(ArrayId::SsmZ(self.layer_index));+ let state_arr = state.array(ArrayId::SsmState(self.layer_index));+ let out = state.array(ArrayId::AttentionOutput);- let x = &base_arrays[0];- let b = &base_arrays[1];- let c = &base_arrays[2];- let dt = &base_arrays[3];- let z = &base_arrays[4];- let state_arr = &base_arrays[5];- let out = &base_arrays[6];-- let skip_rc = self.skip_connection_weight.buffer();- let skip_borrow = skip_rc.borrow();self.ssd_prefill.encode(encoder,SSDPrefillArguments {@@ -315,7 +287,7 @@ impl<B: Backend> MambaMixer<B> {dt: dt.buffer().borrow().deref(),b: b.buffer().borrow().deref(),c: c.buffer().borrow().deref(),- d: skip_borrow.deref(),+ d: self.skip_connection_weight.buffer().borrow().deref(),z: z.buffer().borrow().deref(),state: state_arr.buffer().borrow_mut().deref_mut(),y: out.buffer().borrow_mut().deref_mut(),@@ -339,25 +311,13 @@ impl<B: Backend> MambaMixer<B> {encoder: &mut Encoder<B>,suffix_length: usize,) {- let arrays = state.arrays(&[- ArrayId::SsmX(self.layer_index),- ArrayId::SsmB(self.layer_index),- ArrayId::SsmC(self.layer_index),- ArrayId::SsmDt(self.layer_index),- ArrayId::SsmZ(self.layer_index),- ArrayId::SsmState(self.layer_index),- ArrayId::AttentionOutput,- ]);- let x = &arrays[0];- let b = &arrays[1];- let c = &arrays[2];- let dt = &arrays[3];- let z = &arrays[4];- let state_arr = &arrays[5];- let y = &arrays[6];-- let skip_rc = self.skip_connection_weight.buffer();- let skip_borrow = skip_rc.borrow();+ let x = state.array(ArrayId::SsmX(self.layer_index));+ let b = state.array(ArrayId::SsmB(self.layer_index));+ let c = state.array(ArrayId::SsmC(self.layer_index));+ let dt = state.array(ArrayId::SsmDt(self.layer_index));+ let z = state.array(ArrayId::SsmZ(self.layer_index));+ let state_arr = state.array(ArrayId::SsmState(self.layer_index));+ let y = state.array(ArrayId::AttentionOutput);let h = self.config.num_heads as u32;let g = self.config.num_groups as u32;@@ -376,7 +336,7 @@ impl<B: Backend> MambaMixer<B> {dt.buffer().borrow().deref(),b.buffer().borrow().deref(),c.buffer().borrow().deref(),- skip_borrow.deref(),+ self.skip_connection_weight.buffer().borrow().deref(),z.buffer().borrow().deref(),None::<&B::Buffer>,y.buffer().borrow_mut().deref_mut(),diff --git a/crates/uzu/src/encodable_block/mlp/dense.rs b/crates/uzu/src/encodable_block/mlp/dense.rsindex 70a50712e..2d6558228 100644--- a/crates/uzu/src/encodable_block/mlp/dense.rs+++ b/crates/uzu/src/encodable_block/mlp/dense.rs@@ -38,20 +38,12 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {self.up.encode(state, encoder)?;// Gate act+mul (fused_up -> hidden)- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);- let fused = &arrays[0];- let hidden = &arrays[1];- let m = fused.shape()[0] as i32;- let fused_buf_rc = fused.buffer();- let fused_buf_borrow = fused_buf_rc.borrow();- let hidden_buf_rc = hidden.buffer();- let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();- self.gate- .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)- .expect("Failed to encode MLP activation/mul kernel");- drop(hidden_buf_borrow);- drop(fused_buf_borrow);- drop(arrays);+ {+ let fused = state.array(ArrayId::MlpFusedUp);+ let hidden = state.array(ArrayId::MlpHidden);+ let m = fused.shape()[0] as i32;+ self.gate.encode(encoder, fused.buffer().borrow().deref(), hidden.buffer().borrow_mut().deref_mut(), m)?;+ }// Downself.down.encode(state, encoder)?;diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rsindex 6ab607cd7..1f2da2452 100644--- a/crates/uzu/src/encodable_block/mlp/moe.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -10,7 +10,6 @@ use thiserror::Error;use crate::{DataType,- array::Array,backends::common::{Backend, Encoder, Kernels,gpu_types::ActivationType,@@ -200,53 +199,27 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {encoder: &mut Encoder<B>,) -> Result<(), B::Error> {let suffix_length = state.active_suffix_length();- let arrays = state.arrays(&[- ArrayId::Main,- ArrayId::MoeTopkIds,- ArrayId::MoeTopkProbs,- ArrayId::MoeOffsets,- ArrayId::MoeSumK,- ArrayId::MoeBucketedTokenIds,- ArrayId::MoeBucketedProbs,- ArrayId::MoeXPerm,- ArrayId::MoeTok2Row,- ArrayId::MoeYPartial,- ArrayId::MoeTileCounts,- ArrayId::MoeTileOffsets,- ArrayId::MoeTileMap,- ArrayId::MoeTotalTiles,- ArrayId::MoeDispatchArgs,- ArrayId::MoeScatterPartials,- ArrayId::MoeScatterBlockBases,- ArrayId::MoeBlockAlloc,- ArrayId::MoeHidden,- ArrayId::MoeTwoPassRowExpertMap,- ]);-- let clone_buffer = |array: &Array<B>| -> Rc<RefCell<B::Buffer>> { array.buffer() };-- let mut array_iter = arrays.iter();- let main_buf = clone_buffer(array_iter.next().unwrap());- let topk_ids_buf = clone_buffer(array_iter.next().unwrap());- let topk_probs_buf = clone_buffer(array_iter.next().unwrap());- let offsets_buf = clone_buffer(array_iter.next().unwrap());- let sumk_buf = clone_buffer(array_iter.next().unwrap());- let bucketed_ids_buf = clone_buffer(array_iter.next().unwrap());- let bucketed_probs_buf = clone_buffer(array_iter.next().unwrap());- let x_perm_buf = clone_buffer(array_iter.next().unwrap());- let tok2row_buf = clone_buffer(array_iter.next().unwrap());- let y_partial_buf = clone_buffer(array_iter.next().unwrap());- let tile_counts_buf = clone_buffer(array_iter.next().unwrap());- let tile_offsets_buf = clone_buffer(array_iter.next().unwrap());- let tile_map_buf = clone_buffer(array_iter.next().unwrap());- let total_tiles_buf = clone_buffer(array_iter.next().unwrap());- let dispatch_args_buf = clone_buffer(array_iter.next().unwrap());- let partials_buf = clone_buffer(array_iter.next().unwrap());- let block_bases_buf = clone_buffer(array_iter.next().unwrap());- let block_alloc_buf = clone_buffer(array_iter.next().unwrap());- let hidden_buf = clone_buffer(array_iter.next().unwrap());- let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());- debug_assert!(array_iter.next().is_none());++ let main_buf = state.array(ArrayId::Main).buffer();+ let topk_ids_buf = state.array(ArrayId::MoeTopkIds).buffer();+ let topk_probs_buf = state.array(ArrayId::MoeTopkProbs).buffer();+ let offsets_buf = state.array(ArrayId::MoeOffsets).buffer();+ let sumk_buf = state.array(ArrayId::MoeSumK).buffer();+ let bucketed_ids_buf = state.array(ArrayId::MoeBucketedTokenIds).buffer();+ let bucketed_probs_buf = state.array(ArrayId::MoeBucketedProbs).buffer();+ let x_perm_buf = state.array(ArrayId::MoeXPerm).buffer();+ let tok2row_buf = state.array(ArrayId::MoeTok2Row).buffer();+ let y_partial_buf = state.array(ArrayId::MoeYPartial).buffer();+ let tile_counts_buf = state.array(ArrayId::MoeTileCounts).buffer();+ let tile_offsets_buf = state.array(ArrayId::MoeTileOffsets).buffer();+ let tile_map_buf = state.array(ArrayId::MoeTileMap).buffer();+ let total_tiles_buf = state.array(ArrayId::MoeTotalTiles).buffer();+ let dispatch_args_buf = state.array(ArrayId::MoeDispatchArgs).buffer();+ let partials_buf = state.array(ArrayId::MoeScatterPartials).buffer();+ let block_bases_buf = state.array(ArrayId::MoeScatterBlockBases).buffer();+ let block_alloc_buf = state.array(ArrayId::MoeBlockAlloc).buffer();+ let hidden_buf = state.array(ArrayId::MoeHidden).buffer();+ let row_expert_map_buf = state.array(ArrayId::MoeTwoPassRowExpertMap).buffer();let e = self.moe_config.num_routed_experts;let k = self.moe_config.num_active_routed_experts;diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rsindex 8ee3594ca..fcf34f55f 100644--- a/crates/uzu/src/encodable_block/pooling.rs+++ b/crates/uzu/src/encodable_block/pooling.rs@@ -66,9 +66,8 @@ impl<B: Backend> Pooling<B> {let batch_size = 1;let seq_len = state.aux_buffers_suffix_length();- let arrays = state.arrays(&[ArrayId::Main, ArrayId::ClassifierPooling]);- let main_array = &arrays[0];- let pooling_array = &arrays[1];+ let main_array = state.array(ArrayId::Main);+ let pooling_array = state.array(ArrayId::ClassifierPooling);self.pooling_kernel.encode(main_array.buffer().borrow().deref(),diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rsindex 3bc307c26..bc1de7ffb 100644--- a/crates/uzu/src/encodable_block/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -126,10 +126,8 @@ impl<B: Backend> QKNorm<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let qkv_binding = state.arrays(&[self.qkv_array_id]);- let qkv_array = &qkv_binding[0];- let qkv_shape = qkv_array.shape().to_vec();- let batch_size = qkv_shape[0] as u32;+ let qkv_array = state.array(self.qkv_array_id);+ let batch_size = qkv_array.shape()[0] as u32;// Process query normalization if configuredif let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 9a6784eb5..e70ea0b28 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -87,14 +87,12 @@ impl<B: Backend> RMSNorm<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let input_binding = state.arrays(&[self.input_array_id]);- let output_binding = state.arrays(&[self.output_array_id]);+ let input_array = state.array(self.input_array_id);+ let output_array = state.array(self.output_array_id);- let input_array = &input_binding[0];- let input_shape = input_array.shape().to_vec();- let output_array = &output_binding[0];+ let suffix_length = input_array.shape()[0];+ let element_count = input_array.shape()[1];- let suffix_length = input_shape[0];let input_elem_size = input_array.data_type().size_in_bytes();let output_elem_size = output_array.data_type().size_in_bytes();@@ -103,15 +101,16 @@ impl<B: Backend> RMSNorm<B> {} else {(0, state.active_suffix_length())};+let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));if batch_len == 0 {return Ok(());}- let row_size_in_bytes = input_shape[1] * input_elem_size;+ let row_size_in_bytes = element_count * input_elem_size;let input_offset = batch_start * row_size_in_bytes;- let output_row_size_in_bytes = input_shape[1] * output_elem_size;+ let output_row_size_in_bytes = element_count * output_elem_size;let output_offset = batch_start * output_row_size_in_bytes;let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());@@ -122,7 +121,7 @@ impl<B: Backend> RMSNorm<B> {self.scales_buffer.borrow().deref(),(output_array.buffer().borrow_mut().deref_mut(), output_offset),batch_len as u32,- input_shape[1] as u32,+ element_count as u32,self.config.epsilon,self.config.scale_offset.unwrap_or(0.0),self.config.upcast_mode == UpcastMode::FullLayer,diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rsindex f2832980b..1c63789af 100644--- a/crates/uzu/src/encodable_block/rope.rs+++ b/crates/uzu/src/encodable_block/rope.rs@@ -33,54 +33,28 @@ impl<B: Backend> Rope<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let (suffix_length, num_heads, head_dim, num_groups, rope_dim, rope_max_seq_len) = {- let qkv_binding = state.arrays(&[ArrayId::QKV]);- let qkv_array = &qkv_binding[0];- let suffix_length = qkv_array.shape()[0];+ let token_positions = state.array(ArrayId::TokenPositions);+ let qkv = state.array(ArrayId::QKV);+ let cosines = state.array(ArrayId::RopeCosines(self.rope_type));+ let sines = state.array(ArrayId::RopeSines(self.rope_type));+ let rotated_queries = state.array(ArrayId::RotatedQueries);+ let rotated_keys = state.array(ArrayId::RotatedKeys);- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);- let queries_array = &queries_binding[0];- let num_heads = queries_array.shape()[0];- let head_dim = queries_array.shape()[2];+ let suffix_length = qkv.shape()[0];- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let keys_array = &keys_binding[0];- let num_groups = keys_array.shape()[0];+ let rope_max_seq_len = cosines.shape()[0];+ let rope_dim = cosines.shape()[1];- let cos_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);- let cos_array = &cos_binding[0];- let cos_shape = cos_array.shape();- let rope_max_seq_len = cos_shape[0];- let rope_dim = cos_shape[1];+ let num_heads = rotated_queries.shape()[0];+ let head_dim = rotated_queries.shape()[2];- (suffix_length, num_heads, head_dim, num_groups, rope_dim, rope_max_seq_len)- };-- let qkv_buffer_binding = state.arrays(&[ArrayId::QKV]);- let qkv = &qkv_buffer_binding[0];-- let token_positions_binding = state.arrays(&[ArrayId::TokenPositions]);- let token_positions = &token_positions_binding[0];-- let query_buffer_binding = state.arrays(&[ArrayId::RotatedQueries]);- let rotated_queries = &query_buffer_binding[0];-- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);- let rotated_keys = &rotated_keys_binding[0];-- let cos_buffer_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);- let rope_cosines = &cos_buffer_binding[0];-- let sin_buffer_binding = state.arrays(&[ArrayId::RopeSines(self.rope_type)]);- let rope_sines = &sin_buffer_binding[0];-- let token_positions_offset = token_positions.offset();+ let num_groups = rotated_keys.shape()[0];self.kernel.encode(qkv.buffer().borrow().deref(),- rope_cosines.buffer().borrow().deref(),- rope_sines.buffer().borrow().deref(),- (token_positions.buffer().borrow().deref(), token_positions_offset),+ cosines.buffer().borrow().deref(),+ sines.buffer().borrow().deref(),+ (token_positions.buffer().borrow().deref(), token_positions.offset()),rotated_queries.buffer().borrow_mut().deref_mut(),rotated_keys.buffer().borrow_mut().deref_mut(),head_dim as u32,diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex df991f479..9f0efd154 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -33,25 +33,15 @@ impl<B: Backend> Sampling<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");-- let logits_binding = state.arrays(&[ArrayId::Logits]);- let logits = &logits_binding[0];-- let logits_shape = logits.shape();let batch_size = state.sampling_length();- if batch_size == 0 {- return Ok(());- }let sampling_start = state.sampling_start();- let vocab_size = logits_shape[1];-- let seeds_binding = state.arrays(&[ArrayId::TokenSeeds]);- let seeds = &seeds_binding[0];+ let sampling_method = state.sampling_method().unwrap();- let output_buffer_ref = state.sampling_output().unwrap();+ let logits = state.array(ArrayId::Logits);+ let seeds = state.array(ArrayId::TokenSeeds);+ let output = state.sampling_output().unwrap();- let sampling_method = state.sampling_method().unwrap();+ let vocab_size = logits.shape()[1];let seeds_offset = seeds.offset() + sampling_start * std::mem::size_of::<u64>();let (bitmask_buffer, bitmask_offset) = state.token_bitmask().map_or((None, 0usize), |bitmask| {@@ -60,26 +50,22 @@ impl<B: Backend> Sampling<B> {(Some(bitmask.buffer()), bitmask_offset)});let bitmask_borrow = bitmask_buffer.as_ref().map(|b| b.borrow());- let logits_buf_rc = logits.buffer();- let mut logits_buf_borrow = logits_buf_rc.borrow_mut();- let seeds_buf_rc = seeds.buffer();- let seeds_buf_borrow = seeds_buf_rc.borrow();- let output_buf_rc = output_buffer_ref.buffer();- let mut output_buf_borrow = output_buf_rc.borrow_mut();- if let Err(e) = self.kernel.encode(- logits_buf_borrow.deref_mut(),- Some(seeds_buf_borrow.deref()),- seeds_offset,- bitmask_borrow.as_deref(),- bitmask_offset,- output_buf_borrow.deref_mut(),- sampling_method,- batch_size,- vocab_size,- encoder,- ) {- panic!("Sampling encoding failed: {:?}", e);- }++ self.kernel+ .encode(+ logits.buffer().borrow_mut().deref_mut(),+ seeds.buffer().borrow().deref(),+ seeds_offset,+ bitmask_borrow.as_deref(),+ bitmask_offset,+ output.buffer().borrow_mut().deref_mut(),+ sampling_method,+ batch_size,+ vocab_size,+ encoder,+ )+ .expect("Sampling encoding failed");+Ok(())}}diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex 1fb66558f..e9d0b7696 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -176,14 +176,10 @@ impl<B: Backend> ShortConvMixer<B> {return;}- let arrays =- state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);- let in_proj = &arrays[0];- let conv_state = &arrays[1];- let out = &arrays[2];-- let weight_buf_rc = self.conv_weight.buffer();- let weight_buf_borrow = weight_buf_rc.borrow();+ let in_proj = state.array(ArrayId::SsmInProj);+ let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));+ let out = state.array(ArrayId::AttentionOutput);+let bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());@@ -210,7 +206,7 @@ impl<B: Backend> ShortConvMixer<B> {self.short_conv_prefill.encode(&padded_buf,in_proj.buffer().borrow().deref(),- weight_buf_borrow.deref(),+ self.conv_weight.buffer().borrow().deref(),bias_buf_borrow.as_deref(),out.buffer().borrow_mut().deref_mut(),conv_state.buffer().borrow_mut().deref_mut(),@@ -234,19 +230,11 @@ impl<B: Backend> ShortConvMixer<B> {return;}- let arrays = state.arrays(&[- ArrayId::SsmInProj,- ArrayId::TokenParents,- ArrayId::ShortConvState(self.layer_index),- ArrayId::ShortConvSuffixState(self.layer_index),- ArrayId::AttentionOutput,- ]);-- let in_proj = &arrays[0];- let parents = &arrays[1];- let conv_state = &arrays[2];- let suffix_state = &arrays[3];- let out = &arrays[4];+ let in_proj = state.array(ArrayId::SsmInProj);+ let parents = state.array(ArrayId::TokenParents);+ let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));+ let suffix_state = state.array(ArrayId::ShortConvSuffixState(self.layer_index));+ let out = state.array(ArrayId::AttentionOutput);let data_type: DataType = self.config.in_projection_config.activation_precision().into();let elem_bytes = data_type.size_in_bytes();@@ -260,14 +248,12 @@ impl<B: Backend> ShortConvMixer<B> {let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;let base_state_offset = conv_state.offset();let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();- let trie_weight_buf_rc = self.conv_weight.buffer();- let trie_weight_buf_borrow = trie_weight_buf_rc.borrow();let trie_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());let trie_bias_buf_borrow = trie_bias_buf_rc.as_ref().map(|rc| rc.borrow());self.short_conv_trie.encode((in_proj.buffer().borrow().deref(), in_proj_offset),- trie_weight_buf_borrow.deref(),+ self.conv_weight.buffer().borrow().deref(),trie_bias_buf_borrow.as_deref(),(conv_state.buffer().borrow().deref(), base_state_offset),(parents.buffer().borrow().deref(), parents_offset),@@ -292,22 +278,19 @@ impl<B: Backend> ShortConvMixer<B> {return;}- let arrays =- state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);- let in_proj = &arrays[0];- let conv_state = &arrays[1];- let out = &arrays[2];+ let in_proj = state.array(ArrayId::SsmInProj);+ let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));+ let out = state.array(ArrayId::AttentionOutput);- let decode_weight_buf_rc = self.conv_weight.buffer();- let decode_weight_buf_borrow = decode_weight_buf_rc.borrow();let decode_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());let decode_bias_buf_borrow = decode_bias_buf_rc.as_ref().map(|rc| rc.borrow());+let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);self.short_conv_decode.encode(in_proj.buffer().borrow().deref(),- decode_weight_buf_borrow.deref(),+ self.conv_weight.buffer().borrow().deref(),decode_bias_buf_borrow.as_deref(),None::<&B::Buffer>,out.buffer().borrow_mut().deref_mut(),diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rsindex 7d750491f..2320c2c4a 100644--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs@@ -13,19 +13,23 @@ use crate::{pub struct TensorAddSwap<B: Backend> {kernel: <B::Kernels as Kernels>::TensorAddSwapKernel,- argument_arrays: Box<[ArrayId]>,+ skip_array_id: ArrayId,+ main_array_id: ArrayId,}impl<B: Backend> TensorAddSwap<B> {pub fn new(context: &B::Context,data_type: DataType,- argument_arrays: Box<[ArrayId]>,+ skip_array_id: ArrayId,+ main_array_id: ArrayId,) -> Result<Self, B::Error> {let kernel = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)?;+Ok(Self {kernel,- argument_arrays,+ skip_array_id,+ main_array_id,})}@@ -34,13 +38,10 @@ impl<B: Backend> TensorAddSwap<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let arrays = state.arrays(&self.argument_arrays);- assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");-- let length = arrays[0].num_elements();+ let skip_array = state.array(self.skip_array_id);+ let main_array = state.array(self.main_array_id);- let skip_array = &arrays[0];- let main_array = &arrays[1];+ let length = skip_array.num_elements();self.kernel.encode(skip_array.buffer().borrow_mut().deref_mut(),diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rsindex 708bbe620..69e6c9117 100644--- a/crates/uzu/src/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/encodable_block/tensor_copy.rs@@ -13,19 +13,23 @@ use crate::{pub struct TensorCopy<B: Backend> {kernel: <B::Kernels as Kernels>::TensorCopyKernel,- argument_arrays: Box<[ArrayId]>,+ source_array: ArrayId,+ destination_array: ArrayId,}impl<B: Backend> TensorCopy<B> {pub fn new(context: &B::Context,data_type: DataType,- argument_arrays: Box<[ArrayId]>,+ source_array: ArrayId,+ destination_array: ArrayId,) -> Result<Self, B::Error> {let kernel = <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type)?;+Ok(Self {kernel,- argument_arrays,+ source_array,+ destination_array,})}@@ -34,13 +38,10 @@ impl<B: Backend> TensorCopy<B> {state: &mut ForwardPassState<B>,encoder: &mut Encoder<B>,) -> Result<(), B::Error> {- let arrays = state.arrays(&self.argument_arrays);- assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");-- let length = arrays[0].num_elements();+ let source_array = state.array(self.source_array);+ let destination_array = state.array(self.destination_array);- let source_array = &arrays[0];- let destination_array = &arrays[1];+ let length = source_array.num_elements();self.kernel.encode(source_array.buffer().borrow().deref(),diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex f2b84ea6f..1318e2570 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -658,13 +658,6 @@ impl<B: Backend> ForwardPassState<B> {// Public API Methods (formerly trait methods)// ========================================================================- pub fn arrays(- &self,- ids: &[ArrayId],- ) -> Box<[Array<B>]> {- ids.iter().map(|id| self.array(*id)).collect()- }-pub fn hashmaps(&self,ids: &[HashMapId],@@ -749,13 +742,13 @@ impl<B: Backend> ForwardPassState<B> {source_array_id: ArrayId,destination_array: Array<B>,) {- let source_ref = self.arrays(&[source_array_id])[0].clone();+ let source_array = self.array(source_array_id);- let src_buf_rc = source_ref.buffer();+ let src_buf_rc = source_array.buffer();let dst_buf_rc = destination_array.buffer();let copy_size_bytes = destination_array.size();- debug_assert_eq!(destination_array.size(), source_ref.size());+ debug_assert_eq!(destination_array.size(), source_array.size());encoder.encode_copy(src_buf_rc.borrow().deref(),diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rsindex 3c0b43a59..ae287aa1c 100644--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs@@ -935,7 +935,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {model_dim,})?;- let main = state.arrays(&[ArrayId::Main])[0].clone();+ let main = state.array(ArrayId::Main);let bytes_per_element = main.data_type().size_in_bytes();let row_offset = (token_count - 1).checked_mul(model_dim)@@ -975,7 +975,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {model_dim,})?;- let main = state.arrays(&[ArrayId::Main])[0].clone();+ let main = state.array(ArrayId::Main);if override_embedding.shape() != [1, model_dim] || override_embedding.data_type() != main.data_type() {return Err(TtsModelConfigError::OverrideEmbeddingTensorMismatch {expected_shape: [1, model_dim].into(),@@ -1022,7 +1022,7 @@ impl<B: Backend> TokenDecoderRunner<B> {total_len,})?;- let main = state.arrays(&[ArrayId::Main])[0].clone();+ let main = state.array(ArrayId::Main);let bias = &self.single_override_embedding;if bias.shape() != [1, model_dim] || bias.data_type() != main.data_type() {return Err(TtsModelConfigError::AddScaleBiasTensorMismatch {diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 248b9cac8..9dd4de189 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -292,19 +292,20 @@ impl<B: Backend> TraceValidator<B> {};for index in transformer_layers {- let arrays = state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);+ let keys = state.array(ArrayId::Keys(index));+ let values = state.array(ArrayId::Values(index));if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.keys", index),- metrics: Self::validate_array(data_type, &expected, &arrays[0], Some(ArrayTransform::KVCacheSlice)),+ metrics: Self::validate_array(data_type, &expected, &keys, Some(ArrayTransform::KVCacheSlice)),});}if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.values", index),- metrics: Self::validate_array(data_type, &expected, &arrays[1], Some(ArrayTransform::KVCacheSlice)),+ metrics: Self::validate_array(data_type, &expected, &values, Some(ArrayTransform::KVCacheSlice)),});}}@@ -316,7 +317,8 @@ impl<B: Backend> TraceValidator<B> {};for index in ssm_layers {- let arrays = state.arrays(&[ArrayId::SsmConvState(index), ArrayId::SsmState(index)]);+ let conv_state = state.array(ArrayId::SsmConvState(index));+ let ssm_state = state.array(ArrayId::SsmState(index));for path in [format!("updated_state.{}.conv_state", index),@@ -328,7 +330,7 @@ impl<B: Backend> TraceValidator<B> {metrics: Self::validate_array(data_type,&expected,- &arrays[0],+ &conv_state,Some(ArrayTransform::SsmConvState),),});@@ -342,7 +344,7 @@ impl<B: Backend> TraceValidator<B> {if let Ok(expected) = traces_view.leaf_array(&path) {results.push(TracerValidationResult {name: path,- metrics: Self::validate_array(data_type, &expected, &arrays[1], None),+ metrics: Self::validate_array(data_type, &expected, &ssm_state, None),});}}diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rsindex 952f4dfdc..4d4fd78aa 100644--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs@@ -70,6 +70,11 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {.new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");+ let seeds_buffer = context+ .device+ .new_buffer_with_data(&vec![0; 16], MTLResourceOptions::STORAGE_MODE_SHARED)+ .expect("Failed to create buffer");+let mut output_buffer = context.device.new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)@@ -79,7 +84,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {kernel.encode(&mut logits_buffer,- None,+ &seeds_buffer,0,None,0,@@ -170,7 +175,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {kernel.encode(&mut logits_buf,- Some(&seeds_buf),+ &seeds_buf,0,None,0,@@ -282,7 +287,7 @@ fn test_categorical_sampling() {kernel.encode(&mut logits_buffer,- Some(&seeds_buffer),+ &seeds_buffer,0,None,0,@@ -419,7 +424,7 @@ fn test_categorical_sampling_statistical() {kernel.encode(&mut logits_buffer,- Some(&seeds_buffer),+ &seeds_buffer,0,None,0,@@ -521,7 +526,7 @@ fn perf_categorical_128k_vocab() {kernel.encode(&mut logits_buf,- Some(&seeds_buf),+ &seeds_buf,0,None,0,@@ -746,7 +751,7 @@ fn test_minp_sampling_exact_match(kernel.encode(&mut logits_buf,- Some(&seeds_buf),+ &seeds_buf,0,None,0,diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rsindex b35d4facb..0bd3c6b46 100644--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs@@ -283,7 +283,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(kernel.encode(logits_array.buffer().borrow_mut().deref_mut(),- Some(seeds_array.buffer().borrow().deref()),+ seeds_array.buffer().borrow().deref(),0,None,0,@@ -388,7 +388,7 @@ fn test_topp_sampling_statistical_large() {kernel.encode(logits_array.buffer().borrow_mut().deref_mut(),- Some(seeds_array.buffer().borrow().deref()),+ seeds_array.buffer().borrow().deref(),0,None,0,@@ -465,7 +465,7 @@ fn perf_topp_128k_vocab() {kernel.encode(logits_array.buffer().borrow_mut().deref_mut(),- Some(seeds_array.buffer().borrow().deref()),+ seeds_array.buffer().borrow().deref(),0,None,0,