Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

array getter cleanup (#291)

uuuvn4cfcad47172b
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index b89f34fb7..42e39e5ca 100644
--- a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -353,7 +353,7 @@ impl StructuredAudioCodecGraph {
1,
);
- let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let main = state.array(ArrayId::Main);
let main_output = {
if main.shape() != [frames, self.input_dim] {
return Err(AudioError::Runtime(format!(
@@ -462,7 +462,7 @@ impl StructuredAudioCodecGraph {
1,
);
- let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let main = state.array(ArrayId::Main);
let main_output = {
if main.shape() != [active_len, self.input_dim] {
return Err(AudioError::Runtime(format!(
diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rs
index 491431e61..b55c6214d 100644
--- a/crates/uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/uzu/src/backends/common/kernel/sampling.rs
@@ -58,8 +58,6 @@ pub enum SamplingError<B: Backend> {
BatchSizeExceeded(usize, usize),
#[error("Vocab size {0} exceeds maximum {1}")]
VocabSizeExceeded(usize, usize),
- #[error("Stochastic sampling encode must have a seed")]
- StochasticWithoutSeed,
}
impl<B: Backend> SamplingKernel<B> {
@@ -144,7 +142,7 @@ impl<B: Backend> SamplingKernel<B> {
pub fn encode(
&self,
mut logits_buffer: &mut B::Buffer,
- seeds_buffer: Option<&B::Buffer>,
+ seeds_buffer: &B::Buffer,
seeds_offset: usize,
bitmask_buffer: Option<&B::Buffer>,
bitmask_offset: usize,
@@ -239,7 +237,7 @@ impl<B: Backend> SamplingKernel<B> {
self.gumbel.encode(
None::<&B::Buffer>,
- (seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),
+ (seeds_buffer, seeds_offset),
logits_buffer.deref_mut(),
batch_size as u32,
vocab_size as u32,
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 6b9f0bd02..f8c34c890 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -186,25 +186,13 @@ impl<B: Backend> Classifier<B> {
&self,
state: &ForwardPassState<B>,
) -> Result<Box<[f32]>, Error> {
- let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);
- let logits_array = &logits_arrays[0];
-
let num_labels = self.context.model_config.model_config.num_labels;
- let buffer = logits_array.as_bytes();
-
- match logits_array.data_type() {
- DataType::F32 => {
- let slice: &[f32] = bytemuck::cast_slice(buffer);
- Ok(slice[..num_labels].into())
- },
- DataType::F16 => {
- let slice: &[half::f16] = bytemuck::cast_slice(buffer);
- Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())
- },
- DataType::BF16 => {
- let slice: &[half::bf16] = bytemuck::cast_slice(buffer);
- Ok(slice[..num_labels].iter().map(|&x| x.to_f32()).collect::<Vec<_>>().into_boxed_slice())
- },
+ let logits = state.array(ArrayId::ClassifierPredictionHeadLogits).view(&[num_labels]);
+
+ match logits.data_type() {
+ DataType::F32 => Ok(logits.as_slice::<f32>().into()),
+ DataType::F16 => Ok(logits.as_slice::<half::f16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),
+ DataType::BF16 => Ok(logits.as_slice::<half::bf16>().iter().map(|&x| x.to_f32()).collect::<Box<[_]>>()),
_ => Err(Error::UnableToDecodeText),
}
}
diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rs
index 27177bea0..55598deec 100644
--- a/crates/uzu/src/encodable_block/activation.rs
+++ b/crates/uzu/src/encodable_block/activation.rs
@@ -42,9 +42,8 @@ impl<B: Backend> Activation<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
- let input_array = &arrays[0];
- let output_array = &arrays[1];
+ let input_array = state.array(self.input_array_id);
+ let output_array = state.array(self.output_array_id);
let n = input_array.shape().iter().product::<usize>();
if self.activation.act_type() == ActivationType::IDENTITY {
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index 2c33295d5..87f65dfb0 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -149,30 +149,23 @@ impl<B: Backend> Attention<B> {
parameters: &EncodingParameters,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {
- let qkv_binding = state.arrays(&[ArrayId::QKV]);
- let qkv_array = &qkv_binding[0];
- let suffix_length = qkv_array.shape()[0];
-
- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let queries_array = &queries_binding[0];
- let num_heads = queries_array.shape()[0];
- let head_dim = queries_array.shape()[2];
-
- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let keys_array = &keys_binding[0];
- let num_groups = keys_array.shape()[0];
-
- let max_sequence_length = if let Some(_kv) = state.cache_layers() {
- let key_cache_binding = state.arrays(&[ArrayId::Keys(self.layer_index)]);
- let key_cache_array = &key_cache_binding[0];
- key_cache_array.shape()[1]
- } else {
- // For classifiers without KV cache, max_sequence_length is just suffix_length
- suffix_length
- };
-
- (suffix_length, num_heads, head_dim, num_groups, max_sequence_length)
+ let qkv_array = state.array(ArrayId::QKV);
+ let queries_array = state.array(ArrayId::RotatedQueries);
+ let rotated_keys_array = state.array(ArrayId::RotatedKeys);
+ let attention_output_array = state.array(ArrayId::AttentionOutput);
+ let partials_array = state.array(ArrayId::AttentionPartials);
+ let sums_array = state.array(ArrayId::AttentionSums);
+ let maxs_array = state.array(ArrayId::AttentionMaxs);
+
+ let suffix_length = qkv_array.shape()[0];
+ let num_heads = queries_array.shape()[0];
+ let head_dim = queries_array.shape()[2];
+ let num_groups = rotated_keys_array.shape()[0];
+ let max_sequence_length = if state.cache_layers().is_some() {
+ state.array(ArrayId::Keys(self.layer_index)).shape()[1]
+ } else {
+ // For classifiers without KV cache, max_sequence_length is just suffix_length
+ suffix_length
};
let (segment_prefix_length, window_length) = if let Some(cache_layers) = state.cache_layers() {
@@ -202,45 +195,32 @@ impl<B: Backend> Attention<B> {
}
let variant = self.select_variant(gemm_enabled, suffix_length, head_dim, sequence_length, use_mask);
- let rotated_queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let qkv_binding = state.arrays(&[ArrayId::QKV]);
let attention_bias_binding = state.hashmaps(&[HashMapId::AttentionBias]);
- let attention_output_binding = state.arrays(&[ArrayId::AttentionOutput]);
- let gate_binding = self.gate_kernel.as_ref().map(|_| state.arrays(&[ArrayId::Gate]));
+ let gate_array = self.gate_kernel.as_ref().map(|_| state.array(ArrayId::Gate));
let mask_kv_seq_stride = 1;
let mask_q_seq_stride = sequence_length as i32;
let mask_head_stride = 0;
- let sinks_binding = if self.has_sinks {
- Some(state.arrays(&[ArrayId::AttentionSinks(self.layer_index)]))
- } else {
- None
- };
+ let sinks_array = self.has_sinks.then(|| state.array(ArrayId::AttentionSinks(self.layer_index)));
- let rotated_keys_array = &rotated_keys_binding[0];
let rotated_keys_buf_rc = rotated_keys_array.buffer();
let mut rotated_keys_buf_borrow = rotated_keys_buf_rc.borrow_mut();
- let qkv_array = &qkv_binding[0];
let qkv_buf_rc = qkv_array.buffer();
let qkv_buf_borrow = qkv_buf_rc.borrow();
// Get KV cache buffers only if KV cache exists (LLM mode)
let has_kv_cache = state.cache_layers().is_some();
- let key_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Keys(self.layer_index)]));
- let key_cache_array = key_cache_binding.as_ref().map(|b| &b[0]);
+ let key_cache_array = has_kv_cache.then(|| state.array(ArrayId::Keys(self.layer_index)));
let key_cache_buf_rc = key_cache_array.as_ref().map(|a| a.buffer());
let mut key_cache_buf_borrow = key_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());
- let value_cache_binding = has_kv_cache.then(|| state.arrays(&[ArrayId::Values(self.layer_index)]));
- let value_cache_array = value_cache_binding.as_ref().map(|b| &b[0]);
+ let value_cache_array = has_kv_cache.then(|| state.array(ArrayId::Values(self.layer_index)));
let value_cache_buf_rc = value_cache_array.as_ref().map(|a| a.buffer());
let mut value_cache_buf_borrow = value_cache_buf_rc.as_ref().map(|rc| rc.borrow_mut());
- let extracted_values_binding = (!has_kv_cache).then(|| state.arrays(&[ArrayId::ExtractedValues]));
- let extracted_values_array = extracted_values_binding.as_ref().map(|b| &b[0]);
+ let extracted_values_array = (!has_kv_cache).then(|| state.array(ArrayId::ExtractedValues));
let extracted_values_buf_rc = extracted_values_array.as_ref().map(|a| a.buffer());
let mut extracted_values_buf_borrow = extracted_values_buf_rc.as_ref().map(|rc| rc.borrow_mut());
@@ -262,11 +242,9 @@ impl<B: Backend> Attention<B> {
);
}
- let queries_array = &rotated_queries_binding[0];
let queries_buf_rc = queries_array.buffer();
let queries_buf_borrow = queries_buf_rc.borrow();
- let attention_output_array = &attention_output_binding[0];
let attention_output_buf_rc = attention_output_array.buffer();
let mut attention_output_buf_borrow = attention_output_buf_rc.borrow_mut();
@@ -283,24 +261,16 @@ impl<B: Backend> Attention<B> {
let attention_bias_buf_borrow = attention_bias_buf_rc.as_ref().map(|rc| rc.borrow());
let attention_bias_buffer: Option<&B::Buffer> = attention_bias_buf_borrow.as_ref().map(|b| b.deref());
- let partials_binding = state.arrays(&[ArrayId::AttentionPartials]);
- let sums_binding = state.arrays(&[ArrayId::AttentionSums]);
- let maxs_binding = state.arrays(&[ArrayId::AttentionMaxs]);
-
- let partials_array = &partials_binding[0];
let partials_buf_rc = partials_array.buffer();
let mut partials_buf_borrow = partials_buf_rc.borrow_mut();
- let sums_array = &sums_binding[0];
let sums_buf_rc = sums_array.buffer();
let mut sums_buf_borrow = sums_buf_rc.borrow_mut();
- let maxs_array = &maxs_binding[0];
let maxs_buf_rc = maxs_array.buffer();
let mut maxs_buf_borrow = maxs_buf_rc.borrow_mut();
- let sinks_borrow = sinks_binding.as_ref().map(|binding| &binding[0]);
- let sinks_buf_rc = sinks_borrow.as_ref().map(|b| b.buffer());
+ let sinks_buf_rc = sinks_array.as_ref().map(|b| b.buffer());
let sinks_buf_borrow = sinks_buf_rc.as_ref().map(|rc| rc.borrow());
let sinks_buffer: Option<&B::Buffer> = sinks_buf_borrow.as_ref().map(|b| b.deref());
@@ -444,7 +414,7 @@ impl<B: Backend> Attention<B> {
}
if let Some(gate_kernel) = &self.gate_kernel {
- let gate_array = &gate_binding.as_ref().unwrap()[0];
+ let gate_array = gate_array.as_ref().unwrap();
let gate_buf_rc = gate_array.buffer();
let gate_buf_borrow = gate_buf_rc.borrow();
let total_elements = (suffix_length * num_heads * head_dim) as u32;
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index a2344e829..313a603e6 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -48,12 +48,8 @@ impl<B: Backend> ClassifierLayer<B> {
let attention_config = layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
- let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap();
+ let copy_main_to_shortcut_mixer =
+ TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();
let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {
if layer_loader.subtree("pre_mixer_norm").is_ok() {
@@ -134,19 +130,11 @@ impl<B: Backend> ClassifierLayer<B> {
None
};
- let mixer_residual_add = TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap();
+ let mixer_residual_add =
+ TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();
- let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap();
+ let copy_main_to_shortcut_mlp =
+ TensorCopy::<B>::new(ctx, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();
let pre_mlp_norm = Normalization::new(
ctx,
@@ -195,12 +183,8 @@ impl<B: Backend> ClassifierLayer<B> {
)
.expect("Failed to create attention kernel");
- let mlp_residual_add = TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap();
+ let mlp_residual_add =
+ TensorAddSwap::<B>::new(ctx, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();
Self {
layer_index,
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index 0ea8075ed..4c21fc68c 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -493,14 +493,12 @@ impl<B: Backend> Embedding<B> {
) -> Result<(), EmbeddingError<B>> {
let batch_size = state.active_suffix_length() as u32;
- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
-
- let token_ids_array = &arrays[0];
+ let token_ids_array = state.array(ArrayId::TokenIds);
let token_ids_buffer_rc = token_ids_array.buffer();
let token_ids_buffer_borrow = token_ids_buffer_rc.borrow();
let token_ids = token_ids_buffer_borrow.deref();
- let output_array = &arrays[1];
+ let output_array = state.array(ArrayId::Main);
let output_buffer_rc = output_array.buffer();
let mut output_buffer_borrow = output_buffer_rc.borrow_mut();
let output = output_buffer_borrow.deref_mut();
@@ -580,14 +578,12 @@ impl<B: Backend> Embedding<B> {
return Ok(());
}
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
-
- let input_array = &arrays[0];
+ let input_array = state.array(ArrayId::Main);
let input_buffer_rc = input_array.buffer();
let input_buffer_borrow = input_buffer_rc.borrow();
let input = input_buffer_borrow.deref();
- let output_array = &arrays[1];
+ let output_array = state.array(ArrayId::Logits);
let output_buffer_rc = output_array.buffer();
let mut output_buffer_borrow = output_buffer_rc.borrow_mut();
let output = output_buffer_borrow.deref_mut();
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 2fb5461b3..641375307 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -51,12 +51,8 @@ impl<B: Backend> LayerExecutables<B> {
MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
MixerConfig::DeltaNet(config) => config.in_proj_config.activation_precision().into(),
};
- let copy_main_to_shortcut = TensorCopy::<B>::new(
- context,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap();
+ let copy_main_to_shortcut =
+ TensorCopy::<B>::new(context, intermediate_data_type, ArrayId::Main, ArrayId::Shortcut).unwrap();
let pre_attention_norm = RMSNorm::new(
context,
@@ -203,12 +199,8 @@ impl<B: Backend> LayerExecutables<B> {
None
};
- let main_shortcut_add_swap = TensorAddSwap::<B>::new(
- context,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap();
+ let main_shortcut_add_swap =
+ TensorAddSwap::<B>::new(context, intermediate_data_type, ArrayId::Shortcut, ArrayId::Main).unwrap();
let pre_mlp_norm = RMSNorm::new(
context,
diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rs
index 8f6deec5c..e811bdfb5 100644
--- a/crates/uzu/src/encodable_block/layer_norm.rs
+++ b/crates/uzu/src/encodable_block/layer_norm.rs
@@ -73,19 +73,11 @@ impl<B: Backend> LayerNorm<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let input_binding = state.arrays(&[self.input_array_id]);
- let output_binding = state.arrays(&[self.output_array_id]);
+ let input_array = state.array(self.input_array_id);
+ let output_array = state.array(self.output_array_id);
- let input_shape = {
- let input_array = &input_binding[0];
- input_array.shape().to_vec()
- };
-
- let input_array = &input_binding[0];
- let output_array = &output_binding[0];
-
- let batch_size = input_shape[0] as u32;
- let model_dim = input_shape[1] as u32;
+ let batch_size = input_array.shape()[0] as u32;
+ let model_dim = input_array.shape()[1] as u32;
let full_layer = if self.config.upcast_mode == UpcastMode::FullLayer {
1u32
} else {
@@ -94,6 +86,7 @@ impl<B: Backend> LayerNorm<B> {
let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());
let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());
+
self.kernel.encode(
input_buffer_borrow.as_deref(),
self.scales_buffer.borrow().deref(),
diff --git a/crates/uzu/src/encodable_block/linear/full_precision.rs b/crates/uzu/src/encodable_block/linear/full_precision.rs
index 686baffdc..64000ebba 100644
--- a/crates/uzu/src/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs
@@ -131,10 +131,9 @@ impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
- let input_array = &arrays[0];
- let output_array = &arrays[1];
+ let input_array = state.array(self.input_array_id);
+ let output_array = state.array(self.output_array_id);
let bias_borrow = self.bias_buffer.as_ref().map(|b| b.borrow());
self.kernel.borrow_mut().encode(
diff --git a/crates/uzu/src/encodable_block/linear/quantized.rs b/crates/uzu/src/encodable_block/linear/quantized.rs
index f88d95ef5..360617310 100644
--- a/crates/uzu/src/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/encodable_block/linear/quantized.rs
@@ -251,10 +251,10 @@ impl<B: Backend> Linear<B> for QuantizedLinear<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
- let input_array = &arrays[0];
- let output_array = &arrays[1];
+
+ let input_array = state.array(self.input_array_id);
+ let output_array = state.array(self.output_array_id);
let output_buf_rc = output_array.buffer();
let mut output_buf_borrow = output_buf_rc.borrow_mut();
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index 4bd31cca3..f7a5862b1 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -149,20 +149,11 @@ impl<B: Backend> MambaMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) {
- let arrays = state.arrays(&[
- ArrayId::SsmInProj,
- ArrayId::SsmPacked(self.layer_index),
- ArrayId::SsmZ(self.layer_index),
- ArrayId::SsmDt(self.layer_index),
- ]);
-
- let in_proj = &arrays[0];
- let conv_inputs = &arrays[1];
- let gate = &arrays[2];
- let dt = &arrays[3];
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));
+ let gate = state.array(ArrayId::SsmZ(self.layer_index));
+ let dt = state.array(ArrayId::SsmDt(self.layer_index));
- let bias_buf_rc = self.gate_bias.buffer();
- let bias_buf_borrow = bias_buf_rc.borrow();
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let num_heads = self.config.num_heads;
@@ -173,7 +164,7 @@ impl<B: Backend> MambaMixer<B> {
conv_inputs.buffer().borrow_mut().deref_mut(),
gate.buffer().borrow_mut().deref_mut(),
dt.buffer().borrow_mut().deref_mut(),
- bias_buf_borrow.deref(),
+ self.gate_bias.buffer().borrow().deref(),
suffix_length as u32,
total_dim as u32,
conv_dim as u32,
@@ -189,18 +180,11 @@ impl<B: Backend> MambaMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) {
- let arrays = state.arrays(&[
- ArrayId::SsmPacked(self.layer_index),
- ArrayId::SsmConvState(self.layer_index),
- ArrayId::SsmX(self.layer_index),
- ArrayId::SsmB(self.layer_index),
- ArrayId::SsmC(self.layer_index),
- ]);
- let conv_inputs = &arrays[0];
- let conv_state = &arrays[1];
- let x_arr = &arrays[2];
- let b_arr = &arrays[3];
- let c_arr = &arrays[4];
+ let conv_inputs = state.array(ArrayId::SsmPacked(self.layer_index));
+ let conv_state = state.array(ArrayId::SsmConvState(self.layer_index));
+ let x_arr = state.array(ArrayId::SsmX(self.layer_index));
+ let b_arr = state.array(ArrayId::SsmB(self.layer_index));
+ let c_arr = state.array(ArrayId::SsmC(self.layer_index));
let weight_buf_rc = self.conv_weight.buffer();
let weight_buf_borrow = weight_buf_rc.borrow();
@@ -288,26 +272,14 @@ impl<B: Backend> MambaMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) {
- let base_arrays = state.arrays(&[
- ArrayId::SsmX(self.layer_index),
- ArrayId::SsmB(self.layer_index),
- ArrayId::SsmC(self.layer_index),
- ArrayId::SsmDt(self.layer_index),
- ArrayId::SsmZ(self.layer_index),
- ArrayId::SsmState(self.layer_index),
- ArrayId::AttentionOutput,
- ]);
+ let x = state.array(ArrayId::SsmX(self.layer_index));
+ let b = state.array(ArrayId::SsmB(self.layer_index));
+ let c = state.array(ArrayId::SsmC(self.layer_index));
+ let dt = state.array(ArrayId::SsmDt(self.layer_index));
+ let z = state.array(ArrayId::SsmZ(self.layer_index));
+ let state_arr = state.array(ArrayId::SsmState(self.layer_index));
+ let out = state.array(ArrayId::AttentionOutput);
- let x = &base_arrays[0];
- let b = &base_arrays[1];
- let c = &base_arrays[2];
- let dt = &base_arrays[3];
- let z = &base_arrays[4];
- let state_arr = &base_arrays[5];
- let out = &base_arrays[6];
-
- let skip_rc = self.skip_connection_weight.buffer();
- let skip_borrow = skip_rc.borrow();
self.ssd_prefill.encode(
encoder,
SSDPrefillArguments {
@@ -315,7 +287,7 @@ impl<B: Backend> MambaMixer<B> {
dt: dt.buffer().borrow().deref(),
b: b.buffer().borrow().deref(),
c: c.buffer().borrow().deref(),
- d: skip_borrow.deref(),
+ d: self.skip_connection_weight.buffer().borrow().deref(),
z: z.buffer().borrow().deref(),
state: state_arr.buffer().borrow_mut().deref_mut(),
y: out.buffer().borrow_mut().deref_mut(),
@@ -339,25 +311,13 @@ impl<B: Backend> MambaMixer<B> {
encoder: &mut Encoder<B>,
suffix_length: usize,
) {
- let arrays = state.arrays(&[
- ArrayId::SsmX(self.layer_index),
- ArrayId::SsmB(self.layer_index),
- ArrayId::SsmC(self.layer_index),
- ArrayId::SsmDt(self.layer_index),
- ArrayId::SsmZ(self.layer_index),
- ArrayId::SsmState(self.layer_index),
- ArrayId::AttentionOutput,
- ]);
- let x = &arrays[0];
- let b = &arrays[1];
- let c = &arrays[2];
- let dt = &arrays[3];
- let z = &arrays[4];
- let state_arr = &arrays[5];
- let y = &arrays[6];
-
- let skip_rc = self.skip_connection_weight.buffer();
- let skip_borrow = skip_rc.borrow();
+ let x = state.array(ArrayId::SsmX(self.layer_index));
+ let b = state.array(ArrayId::SsmB(self.layer_index));
+ let c = state.array(ArrayId::SsmC(self.layer_index));
+ let dt = state.array(ArrayId::SsmDt(self.layer_index));
+ let z = state.array(ArrayId::SsmZ(self.layer_index));
+ let state_arr = state.array(ArrayId::SsmState(self.layer_index));
+ let y = state.array(ArrayId::AttentionOutput);
let h = self.config.num_heads as u32;
let g = self.config.num_groups as u32;
@@ -376,7 +336,7 @@ impl<B: Backend> MambaMixer<B> {
dt.buffer().borrow().deref(),
b.buffer().borrow().deref(),
c.buffer().borrow().deref(),
- skip_borrow.deref(),
+ self.skip_connection_weight.buffer().borrow().deref(),
z.buffer().borrow().deref(),
None::<&B::Buffer>,
y.buffer().borrow_mut().deref_mut(),
diff --git a/crates/uzu/src/encodable_block/mlp/dense.rs b/crates/uzu/src/encodable_block/mlp/dense.rs
index 70a50712e..2d6558228 100644
--- a/crates/uzu/src/encodable_block/mlp/dense.rs
+++ b/crates/uzu/src/encodable_block/mlp/dense.rs
@@ -38,20 +38,12 @@ impl<B: Backend> Mlp<B> for DenseMlp<B> {
self.up.encode(state, encoder)?;
// Gate act+mul (fused_up -> hidden)
- let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
- let fused = &arrays[0];
- let hidden = &arrays[1];
- let m = fused.shape()[0] as i32;
- let fused_buf_rc = fused.buffer();
- let fused_buf_borrow = fused_buf_rc.borrow();
- let hidden_buf_rc = hidden.buffer();
- let mut hidden_buf_borrow = hidden_buf_rc.borrow_mut();
- self.gate
- .encode(encoder, fused_buf_borrow.deref(), hidden_buf_borrow.deref_mut(), m)
- .expect("Failed to encode MLP activation/mul kernel");
- drop(hidden_buf_borrow);
- drop(fused_buf_borrow);
- drop(arrays);
+ {
+ let fused = state.array(ArrayId::MlpFusedUp);
+ let hidden = state.array(ArrayId::MlpHidden);
+ let m = fused.shape()[0] as i32;
+ self.gate.encode(encoder, fused.buffer().borrow().deref(), hidden.buffer().borrow_mut().deref_mut(), m)?;
+ }
// Down
self.down.encode(state, encoder)?;
diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
index 6ab607cd7..1f2da2452 100644
--- a/crates/uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -10,7 +10,6 @@ use thiserror::Error;
use crate::{
DataType,
- array::Array,
backends::common::{
Backend, Encoder, Kernels,
gpu_types::ActivationType,
@@ -200,53 +199,27 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
let suffix_length = state.active_suffix_length();
- let arrays = state.arrays(&[
- ArrayId::Main,
- ArrayId::MoeTopkIds,
- ArrayId::MoeTopkProbs,
- ArrayId::MoeOffsets,
- ArrayId::MoeSumK,
- ArrayId::MoeBucketedTokenIds,
- ArrayId::MoeBucketedProbs,
- ArrayId::MoeXPerm,
- ArrayId::MoeTok2Row,
- ArrayId::MoeYPartial,
- ArrayId::MoeTileCounts,
- ArrayId::MoeTileOffsets,
- ArrayId::MoeTileMap,
- ArrayId::MoeTotalTiles,
- ArrayId::MoeDispatchArgs,
- ArrayId::MoeScatterPartials,
- ArrayId::MoeScatterBlockBases,
- ArrayId::MoeBlockAlloc,
- ArrayId::MoeHidden,
- ArrayId::MoeTwoPassRowExpertMap,
- ]);
-
- let clone_buffer = |array: &Array<B>| -> Rc<RefCell<B::Buffer>> { array.buffer() };
-
- let mut array_iter = arrays.iter();
- let main_buf = clone_buffer(array_iter.next().unwrap());
- let topk_ids_buf = clone_buffer(array_iter.next().unwrap());
- let topk_probs_buf = clone_buffer(array_iter.next().unwrap());
- let offsets_buf = clone_buffer(array_iter.next().unwrap());
- let sumk_buf = clone_buffer(array_iter.next().unwrap());
- let bucketed_ids_buf = clone_buffer(array_iter.next().unwrap());
- let bucketed_probs_buf = clone_buffer(array_iter.next().unwrap());
- let x_perm_buf = clone_buffer(array_iter.next().unwrap());
- let tok2row_buf = clone_buffer(array_iter.next().unwrap());
- let y_partial_buf = clone_buffer(array_iter.next().unwrap());
- let tile_counts_buf = clone_buffer(array_iter.next().unwrap());
- let tile_offsets_buf = clone_buffer(array_iter.next().unwrap());
- let tile_map_buf = clone_buffer(array_iter.next().unwrap());
- let total_tiles_buf = clone_buffer(array_iter.next().unwrap());
- let dispatch_args_buf = clone_buffer(array_iter.next().unwrap());
- let partials_buf = clone_buffer(array_iter.next().unwrap());
- let block_bases_buf = clone_buffer(array_iter.next().unwrap());
- let block_alloc_buf = clone_buffer(array_iter.next().unwrap());
- let hidden_buf = clone_buffer(array_iter.next().unwrap());
- let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());
- debug_assert!(array_iter.next().is_none());
+
+ let main_buf = state.array(ArrayId::Main).buffer();
+ let topk_ids_buf = state.array(ArrayId::MoeTopkIds).buffer();
+ let topk_probs_buf = state.array(ArrayId::MoeTopkProbs).buffer();
+ let offsets_buf = state.array(ArrayId::MoeOffsets).buffer();
+ let sumk_buf = state.array(ArrayId::MoeSumK).buffer();
+ let bucketed_ids_buf = state.array(ArrayId::MoeBucketedTokenIds).buffer();
+ let bucketed_probs_buf = state.array(ArrayId::MoeBucketedProbs).buffer();
+ let x_perm_buf = state.array(ArrayId::MoeXPerm).buffer();
+ let tok2row_buf = state.array(ArrayId::MoeTok2Row).buffer();
+ let y_partial_buf = state.array(ArrayId::MoeYPartial).buffer();
+ let tile_counts_buf = state.array(ArrayId::MoeTileCounts).buffer();
+ let tile_offsets_buf = state.array(ArrayId::MoeTileOffsets).buffer();
+ let tile_map_buf = state.array(ArrayId::MoeTileMap).buffer();
+ let total_tiles_buf = state.array(ArrayId::MoeTotalTiles).buffer();
+ let dispatch_args_buf = state.array(ArrayId::MoeDispatchArgs).buffer();
+ let partials_buf = state.array(ArrayId::MoeScatterPartials).buffer();
+ let block_bases_buf = state.array(ArrayId::MoeScatterBlockBases).buffer();
+ let block_alloc_buf = state.array(ArrayId::MoeBlockAlloc).buffer();
+ let hidden_buf = state.array(ArrayId::MoeHidden).buffer();
+ let row_expert_map_buf = state.array(ArrayId::MoeTwoPassRowExpertMap).buffer();
let e = self.moe_config.num_routed_experts;
let k = self.moe_config.num_active_routed_experts;
diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rs
index 8ee3594ca..fcf34f55f 100644
--- a/crates/uzu/src/encodable_block/pooling.rs
+++ b/crates/uzu/src/encodable_block/pooling.rs
@@ -66,9 +66,8 @@ impl<B: Backend> Pooling<B> {
let batch_size = 1;
let seq_len = state.aux_buffers_suffix_length();
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::ClassifierPooling]);
- let main_array = &arrays[0];
- let pooling_array = &arrays[1];
+ let main_array = state.array(ArrayId::Main);
+ let pooling_array = state.array(ArrayId::ClassifierPooling);
self.pooling_kernel.encode(
main_array.buffer().borrow().deref(),
diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
index 3bc307c26..bc1de7ffb 100644
--- a/crates/uzu/src/encodable_block/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -126,10 +126,8 @@ impl<B: Backend> QKNorm<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let qkv_binding = state.arrays(&[self.qkv_array_id]);
- let qkv_array = &qkv_binding[0];
- let qkv_shape = qkv_array.shape().to_vec();
- let batch_size = qkv_shape[0] as u32;
+ let qkv_array = state.array(self.qkv_array_id);
+ let batch_size = qkv_array.shape()[0] as u32;
// Process query normalization if configured
if let (Some(query_kernel), Some(query_scales_buffer), Some(query_config)) =
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 9a6784eb5..e70ea0b28 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -87,14 +87,12 @@ impl<B: Backend> RMSNorm<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let input_binding = state.arrays(&[self.input_array_id]);
- let output_binding = state.arrays(&[self.output_array_id]);
+ let input_array = state.array(self.input_array_id);
+ let output_array = state.array(self.output_array_id);
- let input_array = &input_binding[0];
- let input_shape = input_array.shape().to_vec();
- let output_array = &output_binding[0];
+ let suffix_length = input_array.shape()[0];
+ let element_count = input_array.shape()[1];
- let suffix_length = input_shape[0];
let input_elem_size = input_array.data_type().size_in_bytes();
let output_elem_size = output_array.data_type().size_in_bytes();
@@ -103,15 +101,16 @@ impl<B: Backend> RMSNorm<B> {
} else {
(0, state.active_suffix_length())
};
+
let batch_len = batch_len.min(suffix_length.saturating_sub(batch_start));
if batch_len == 0 {
return Ok(());
}
- let row_size_in_bytes = input_shape[1] * input_elem_size;
+ let row_size_in_bytes = element_count * input_elem_size;
let input_offset = batch_start * row_size_in_bytes;
- let output_row_size_in_bytes = input_shape[1] * output_elem_size;
+ let output_row_size_in_bytes = element_count * output_elem_size;
let output_offset = batch_start * output_row_size_in_bytes;
let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());
@@ -122,7 +121,7 @@ impl<B: Backend> RMSNorm<B> {
self.scales_buffer.borrow().deref(),
(output_array.buffer().borrow_mut().deref_mut(), output_offset),
batch_len as u32,
- input_shape[1] as u32,
+ element_count as u32,
self.config.epsilon,
self.config.scale_offset.unwrap_or(0.0),
self.config.upcast_mode == UpcastMode::FullLayer,
diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rs
index f2832980b..1c63789af 100644
--- a/crates/uzu/src/encodable_block/rope.rs
+++ b/crates/uzu/src/encodable_block/rope.rs
@@ -33,54 +33,28 @@ impl<B: Backend> Rope<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let (suffix_length, num_heads, head_dim, num_groups, rope_dim, rope_max_seq_len) = {
- let qkv_binding = state.arrays(&[ArrayId::QKV]);
- let qkv_array = &qkv_binding[0];
- let suffix_length = qkv_array.shape()[0];
+ let token_positions = state.array(ArrayId::TokenPositions);
+ let qkv = state.array(ArrayId::QKV);
+ let cosines = state.array(ArrayId::RopeCosines(self.rope_type));
+ let sines = state.array(ArrayId::RopeSines(self.rope_type));
+ let rotated_queries = state.array(ArrayId::RotatedQueries);
+ let rotated_keys = state.array(ArrayId::RotatedKeys);
- let queries_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let queries_array = &queries_binding[0];
- let num_heads = queries_array.shape()[0];
- let head_dim = queries_array.shape()[2];
+ let suffix_length = qkv.shape()[0];
- let keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let keys_array = &keys_binding[0];
- let num_groups = keys_array.shape()[0];
+ let rope_max_seq_len = cosines.shape()[0];
+ let rope_dim = cosines.shape()[1];
- let cos_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
- let cos_array = &cos_binding[0];
- let cos_shape = cos_array.shape();
- let rope_max_seq_len = cos_shape[0];
- let rope_dim = cos_shape[1];
+ let num_heads = rotated_queries.shape()[0];
+ let head_dim = rotated_queries.shape()[2];
- (suffix_length, num_heads, head_dim, num_groups, rope_dim, rope_max_seq_len)
- };
-
- let qkv_buffer_binding = state.arrays(&[ArrayId::QKV]);
- let qkv = &qkv_buffer_binding[0];
-
- let token_positions_binding = state.arrays(&[ArrayId::TokenPositions]);
- let token_positions = &token_positions_binding[0];
-
- let query_buffer_binding = state.arrays(&[ArrayId::RotatedQueries]);
- let rotated_queries = &query_buffer_binding[0];
-
- let rotated_keys_binding = state.arrays(&[ArrayId::RotatedKeys]);
- let rotated_keys = &rotated_keys_binding[0];
-
- let cos_buffer_binding = state.arrays(&[ArrayId::RopeCosines(self.rope_type)]);
- let rope_cosines = &cos_buffer_binding[0];
-
- let sin_buffer_binding = state.arrays(&[ArrayId::RopeSines(self.rope_type)]);
- let rope_sines = &sin_buffer_binding[0];
-
- let token_positions_offset = token_positions.offset();
+ let num_groups = rotated_keys.shape()[0];
self.kernel.encode(
qkv.buffer().borrow().deref(),
- rope_cosines.buffer().borrow().deref(),
- rope_sines.buffer().borrow().deref(),
- (token_positions.buffer().borrow().deref(), token_positions_offset),
+ cosines.buffer().borrow().deref(),
+ sines.buffer().borrow().deref(),
+ (token_positions.buffer().borrow().deref(), token_positions.offset()),
rotated_queries.buffer().borrow_mut().deref_mut(),
rotated_keys.buffer().borrow_mut().deref_mut(),
head_dim as u32,
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index df991f479..9f0efd154 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -33,25 +33,15 @@ impl<B: Backend> Sampling<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");
-
- let logits_binding = state.arrays(&[ArrayId::Logits]);
- let logits = &logits_binding[0];
-
- let logits_shape = logits.shape();
let batch_size = state.sampling_length();
- if batch_size == 0 {
- return Ok(());
- }
let sampling_start = state.sampling_start();
- let vocab_size = logits_shape[1];
-
- let seeds_binding = state.arrays(&[ArrayId::TokenSeeds]);
- let seeds = &seeds_binding[0];
+ let sampling_method = state.sampling_method().unwrap();
- let output_buffer_ref = state.sampling_output().unwrap();
+ let logits = state.array(ArrayId::Logits);
+ let seeds = state.array(ArrayId::TokenSeeds);
+ let output = state.sampling_output().unwrap();
- let sampling_method = state.sampling_method().unwrap();
+ let vocab_size = logits.shape()[1];
let seeds_offset = seeds.offset() + sampling_start * std::mem::size_of::<u64>();
let (bitmask_buffer, bitmask_offset) = state.token_bitmask().map_or((None, 0usize), |bitmask| {
@@ -60,26 +50,22 @@ impl<B: Backend> Sampling<B> {
(Some(bitmask.buffer()), bitmask_offset)
});
let bitmask_borrow = bitmask_buffer.as_ref().map(|b| b.borrow());
- let logits_buf_rc = logits.buffer();
- let mut logits_buf_borrow = logits_buf_rc.borrow_mut();
- let seeds_buf_rc = seeds.buffer();
- let seeds_buf_borrow = seeds_buf_rc.borrow();
- let output_buf_rc = output_buffer_ref.buffer();
- let mut output_buf_borrow = output_buf_rc.borrow_mut();
- if let Err(e) = self.kernel.encode(
- logits_buf_borrow.deref_mut(),
- Some(seeds_buf_borrow.deref()),
- seeds_offset,
- bitmask_borrow.as_deref(),
- bitmask_offset,
- output_buf_borrow.deref_mut(),
- sampling_method,
- batch_size,
- vocab_size,
- encoder,
- ) {
- panic!("Sampling encoding failed: {:?}", e);
- }
+
+ self.kernel
+ .encode(
+ logits.buffer().borrow_mut().deref_mut(),
+ seeds.buffer().borrow().deref(),
+ seeds_offset,
+ bitmask_borrow.as_deref(),
+ bitmask_offset,
+ output.buffer().borrow_mut().deref_mut(),
+ sampling_method,
+ batch_size,
+ vocab_size,
+ encoder,
+ )
+ .expect("Sampling encoding failed");
+
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index 1fb66558f..e9d0b7696 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -176,14 +176,10 @@ impl<B: Backend> ShortConvMixer<B> {
return;
}
- let arrays =
- state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);
- let in_proj = &arrays[0];
- let conv_state = &arrays[1];
- let out = &arrays[2];
-
- let weight_buf_rc = self.conv_weight.buffer();
- let weight_buf_borrow = weight_buf_rc.borrow();
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));
+ let out = state.array(ArrayId::AttentionOutput);
+
let bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
let bias_buf_borrow = bias_buf_rc.as_ref().map(|rc| rc.borrow());
@@ -210,7 +206,7 @@ impl<B: Backend> ShortConvMixer<B> {
self.short_conv_prefill.encode(
&padded_buf,
in_proj.buffer().borrow().deref(),
- weight_buf_borrow.deref(),
+ self.conv_weight.buffer().borrow().deref(),
bias_buf_borrow.as_deref(),
out.buffer().borrow_mut().deref_mut(),
conv_state.buffer().borrow_mut().deref_mut(),
@@ -234,19 +230,11 @@ impl<B: Backend> ShortConvMixer<B> {
return;
}
- let arrays = state.arrays(&[
- ArrayId::SsmInProj,
- ArrayId::TokenParents,
- ArrayId::ShortConvState(self.layer_index),
- ArrayId::ShortConvSuffixState(self.layer_index),
- ArrayId::AttentionOutput,
- ]);
-
- let in_proj = &arrays[0];
- let parents = &arrays[1];
- let conv_state = &arrays[2];
- let suffix_state = &arrays[3];
- let out = &arrays[4];
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let parents = state.array(ArrayId::TokenParents);
+ let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));
+ let suffix_state = state.array(ArrayId::ShortConvSuffixState(self.layer_index));
+ let out = state.array(ArrayId::AttentionOutput);
let data_type: DataType = self.config.in_projection_config.activation_precision().into();
let elem_bytes = data_type.size_in_bytes();
@@ -260,14 +248,12 @@ impl<B: Backend> ShortConvMixer<B> {
let suffix_state_offset = suffix_state.offset() + sampling_start * self.model_dim * state_stride * elem_bytes;
let base_state_offset = conv_state.offset();
let parents_offset = parents.offset() + sampling_start * std::mem::size_of::<i32>();
- let trie_weight_buf_rc = self.conv_weight.buffer();
- let trie_weight_buf_borrow = trie_weight_buf_rc.borrow();
let trie_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
let trie_bias_buf_borrow = trie_bias_buf_rc.as_ref().map(|rc| rc.borrow());
self.short_conv_trie.encode(
(in_proj.buffer().borrow().deref(), in_proj_offset),
- trie_weight_buf_borrow.deref(),
+ self.conv_weight.buffer().borrow().deref(),
trie_bias_buf_borrow.as_deref(),
(conv_state.buffer().borrow().deref(), base_state_offset),
(parents.buffer().borrow().deref(), parents_offset),
@@ -292,22 +278,19 @@ impl<B: Backend> ShortConvMixer<B> {
return;
}
- let arrays =
- state.arrays(&[ArrayId::SsmInProj, ArrayId::ShortConvState(self.layer_index), ArrayId::AttentionOutput]);
- let in_proj = &arrays[0];
- let conv_state = &arrays[1];
- let out = &arrays[2];
+ let in_proj = state.array(ArrayId::SsmInProj);
+ let conv_state = state.array(ArrayId::ShortConvState(self.layer_index));
+ let out = state.array(ArrayId::AttentionOutput);
- let decode_weight_buf_rc = self.conv_weight.buffer();
- let decode_weight_buf_borrow = decode_weight_buf_rc.borrow();
let decode_bias_buf_rc = self.conv_bias.as_ref().map(|b| b.buffer());
let decode_bias_buf_borrow = decode_bias_buf_rc.as_ref().map(|rc| rc.borrow());
+
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
self.short_conv_decode.encode(
in_proj.buffer().borrow().deref(),
- decode_weight_buf_borrow.deref(),
+ self.conv_weight.buffer().borrow().deref(),
decode_bias_buf_borrow.as_deref(),
None::<&B::Buffer>,
out.buffer().borrow_mut().deref_mut(),
diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rs
index 7d750491f..2320c2c4a 100644
--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs
@@ -13,19 +13,23 @@ use crate::{
pub struct TensorAddSwap<B: Backend> {
kernel: <B::Kernels as Kernels>::TensorAddSwapKernel,
- argument_arrays: Box<[ArrayId]>,
+ skip_array_id: ArrayId,
+ main_array_id: ArrayId,
}
impl<B: Backend> TensorAddSwap<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
- argument_arrays: Box<[ArrayId]>,
+ skip_array_id: ArrayId,
+ main_array_id: ArrayId,
) -> Result<Self, B::Error> {
let kernel = <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type)?;
+
Ok(Self {
kernel,
- argument_arrays,
+ skip_array_id,
+ main_array_id,
})
}
@@ -34,13 +38,10 @@ impl<B: Backend> TensorAddSwap<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let arrays = state.arrays(&self.argument_arrays);
- assert_eq!(arrays.len(), 2, "TensorAddSwap expects exactly 2 arrays");
-
- let length = arrays[0].num_elements();
+ let skip_array = state.array(self.skip_array_id);
+ let main_array = state.array(self.main_array_id);
- let skip_array = &arrays[0];
- let main_array = &arrays[1];
+ let length = skip_array.num_elements();
self.kernel.encode(
skip_array.buffer().borrow_mut().deref_mut(),
diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rs
index 708bbe620..69e6c9117 100644
--- a/crates/uzu/src/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/encodable_block/tensor_copy.rs
@@ -13,19 +13,23 @@ use crate::{
pub struct TensorCopy<B: Backend> {
kernel: <B::Kernels as Kernels>::TensorCopyKernel,
- argument_arrays: Box<[ArrayId]>,
+ source_array: ArrayId,
+ destination_array: ArrayId,
}
impl<B: Backend> TensorCopy<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
- argument_arrays: Box<[ArrayId]>,
+ source_array: ArrayId,
+ destination_array: ArrayId,
) -> Result<Self, B::Error> {
let kernel = <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type)?;
+
Ok(Self {
kernel,
- argument_arrays,
+ source_array,
+ destination_array,
})
}
@@ -34,13 +38,10 @@ impl<B: Backend> TensorCopy<B> {
state: &mut ForwardPassState<B>,
encoder: &mut Encoder<B>,
) -> Result<(), B::Error> {
- let arrays = state.arrays(&self.argument_arrays);
- assert_eq!(arrays.len(), 2, "TensorCopy expects exactly 2 arrays");
-
- let length = arrays[0].num_elements();
+ let source_array = state.array(self.source_array);
+ let destination_array = state.array(self.destination_array);
- let source_array = &arrays[0];
- let destination_array = &arrays[1];
+ let length = source_array.num_elements();
self.kernel.encode(
source_array.buffer().borrow().deref(),
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index f2b84ea6f..1318e2570 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -658,13 +658,6 @@ impl<B: Backend> ForwardPassState<B> {
// Public API Methods (formerly trait methods)
// ========================================================================
- pub fn arrays(
- &self,
- ids: &[ArrayId],
- ) -> Box<[Array<B>]> {
- ids.iter().map(|id| self.array(*id)).collect()
- }
-
pub fn hashmaps(
&self,
ids: &[HashMapId],
@@ -749,13 +742,13 @@ impl<B: Backend> ForwardPassState<B> {
source_array_id: ArrayId,
destination_array: Array<B>,
) {
- let source_ref = self.arrays(&[source_array_id])[0].clone();
+ let source_array = self.array(source_array_id);
- let src_buf_rc = source_ref.buffer();
+ let src_buf_rc = source_array.buffer();
let dst_buf_rc = destination_array.buffer();
let copy_size_bytes = destination_array.size();
- debug_assert_eq!(destination_array.size(), source_ref.size());
+ debug_assert_eq!(destination_array.size(), source_array.size());
encoder.encode_copy(
src_buf_rc.borrow().deref(),
diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rs
index 3c0b43a59..ae287aa1c 100644
--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs
@@ -935,7 +935,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
model_dim,
})?;
- let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let main = state.array(ArrayId::Main);
let bytes_per_element = main.data_type().size_in_bytes();
let row_offset = (token_count - 1)
.checked_mul(model_dim)
@@ -975,7 +975,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
model_dim,
})?;
- let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let main = state.array(ArrayId::Main);
if override_embedding.shape() != [1, model_dim] || override_embedding.data_type() != main.data_type() {
return Err(TtsModelConfigError::OverrideEmbeddingTensorMismatch {
expected_shape: [1, model_dim].into(),
@@ -1022,7 +1022,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
total_len,
})?;
- let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let main = state.array(ArrayId::Main);
let bias = &self.single_override_embedding;
if bias.shape() != [1, model_dim] || bias.data_type() != main.data_type() {
return Err(TtsModelConfigError::AddScaleBiasTensorMismatch {
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 248b9cac8..9dd4de189 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -292,19 +292,20 @@ impl<B: Backend> TraceValidator<B> {
};
for index in transformer_layers {
- let arrays = state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);
+ let keys = state.array(ArrayId::Keys(index));
+ let values = state.array(ArrayId::Values(index));
if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.keys", index),
- metrics: Self::validate_array(data_type, &expected, &arrays[0], Some(ArrayTransform::KVCacheSlice)),
+ metrics: Self::validate_array(data_type, &expected, &keys, Some(ArrayTransform::KVCacheSlice)),
});
}
if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.values", index),
- metrics: Self::validate_array(data_type, &expected, &arrays[1], Some(ArrayTransform::KVCacheSlice)),
+ metrics: Self::validate_array(data_type, &expected, &values, Some(ArrayTransform::KVCacheSlice)),
});
}
}
@@ -316,7 +317,8 @@ impl<B: Backend> TraceValidator<B> {
};
for index in ssm_layers {
- let arrays = state.arrays(&[ArrayId::SsmConvState(index), ArrayId::SsmState(index)]);
+ let conv_state = state.array(ArrayId::SsmConvState(index));
+ let ssm_state = state.array(ArrayId::SsmState(index));
for path in [
format!("updated_state.{}.conv_state", index),
@@ -328,7 +330,7 @@ impl<B: Backend> TraceValidator<B> {
metrics: Self::validate_array(
data_type,
&expected,
- &arrays[0],
+ &conv_state,
Some(ArrayTransform::SsmConvState),
),
});
@@ -342,7 +344,7 @@ impl<B: Backend> TraceValidator<B> {
if let Ok(expected) = traces_view.leaf_array(&path) {
results.push(TracerValidationResult {
name: path,
- metrics: Self::validate_array(data_type, &expected, &arrays[1], None),
+ metrics: Self::validate_array(data_type, &expected, &ssm_state, None),
});
}
}
diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
index 952f4dfdc..4d4fd78aa 100644
--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
@@ -70,6 +70,11 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
.new_buffer_with_data(bytemuck::cast_slice(&test_logits), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
+ let seeds_buffer = context
+ .device
+ .new_buffer_with_data(&vec![0; 16], MTLResourceOptions::STORAGE_MODE_SHARED)
+ .expect("Failed to create buffer");
+
let mut output_buffer = context
.device
.new_buffer(batch_size * std::mem::size_of::<u32>(), MTLResourceOptions::STORAGE_MODE_SHARED)
@@ -79,7 +84,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
kernel
.encode(
&mut logits_buffer,
- None,
+ &seeds_buffer,
0,
None,
0,
@@ -170,7 +175,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
kernel
.encode(
&mut logits_buf,
- Some(&seeds_buf),
+ &seeds_buf,
0,
None,
0,
@@ -282,7 +287,7 @@ fn test_categorical_sampling() {
kernel
.encode(
&mut logits_buffer,
- Some(&seeds_buffer),
+ &seeds_buffer,
0,
None,
0,
@@ -419,7 +424,7 @@ fn test_categorical_sampling_statistical() {
kernel
.encode(
&mut logits_buffer,
- Some(&seeds_buffer),
+ &seeds_buffer,
0,
None,
0,
@@ -521,7 +526,7 @@ fn perf_categorical_128k_vocab() {
kernel
.encode(
&mut logits_buf,
- Some(&seeds_buf),
+ &seeds_buf,
0,
None,
0,
@@ -746,7 +751,7 @@ fn test_minp_sampling_exact_match(
kernel
.encode(
&mut logits_buf,
- Some(&seeds_buf),
+ &seeds_buf,
0,
None,
0,
diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
index b35d4facb..0bd3c6b46 100644
--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
@@ -283,7 +283,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
kernel
.encode(
logits_array.buffer().borrow_mut().deref_mut(),
- Some(seeds_array.buffer().borrow().deref()),
+ seeds_array.buffer().borrow().deref(),
0,
None,
0,
@@ -388,7 +388,7 @@ fn test_topp_sampling_statistical_large() {
kernel
.encode(
logits_array.buffer().borrow_mut().deref_mut(),
- Some(seeds_array.buffer().borrow().deref()),
+ seeds_array.buffer().borrow().deref(),
0,
None,
0,
@@ -465,7 +465,7 @@ fn perf_topp_128k_vocab() {
kernel
.encode(
logits_array.buffer().borrow_mut().deref_mut(),
- Some(seeds_array.buffer().borrow().deref()),
+ seeds_array.buffer().borrow().deref(),
0,
None,
0,