Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Models loading optimizations (#282)

Aleksandr Golokozf4a9f85c28c0
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rs
index 13226241d..5229d6c34 100644
--- a/crates/uzu/src/array.rs
+++ b/crates/uzu/src/array.rs
@@ -228,7 +228,7 @@ pub trait ArrayContextExt {
label: &str,
) -> Array<Self::Backend>;
- fn create_array(
+ fn create_array_zeros(
&self,
shape: &[usize],
data_type: DataType,
diff --git a/crates/uzu/src/audio/nanocodec/runtime.rs b/crates/uzu/src/audio/nanocodec/runtime.rs
index a7b340ce6..0c998470a 100644
--- a/crates/uzu/src/audio/nanocodec/runtime.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime.rs
@@ -324,17 +324,18 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
let kernel = <B::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32)
.map_err(|err| AudioError::Runtime(format!("failed to initialize fsq decode kernel: {err}")))?;
- let mut tokens_array = context.create_array(
+ let mut tokens_array = context.create_array_zeros(
&[batch_size, self.config.num_groups(), frames],
DataType::I32,
"nanocodec_fsq_decode_tokens",
);
tokens_array.as_slice_mut::<i32>().copy_from_slice(&tokens_i32);
- let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");
+ let mut lengths_array =
+ context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");
lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
- let output = context.create_array(
+ let output = context.create_array_zeros(
&[batch_size, self.config.channels(), frames],
DataType::F32,
"nanocodec_fsq_decode_output",
@@ -414,17 +415,17 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {
let kernel = <B::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32)
.map_err(|err| AudioError::Runtime(format!("failed to initialize fsq encode kernel: {err}")))?;
- let mut input = context.create_array(
+ let mut input = context.create_array_zeros(
&[batch_size, self.config.channels(), frames],
DataType::F32,
"nanocodec_fsq_encode_input",
);
input.as_slice_mut::<f32>().copy_from_slice(&padded_input);
- let mut lengths = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
- let tokens = context.create_array(
+ let tokens = context.create_array_zeros(
&[batch_size, self.config.num_groups(), frames],
DataType::I32,
"nanocodec_fsq_encode_tokens",
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs
index 034414d0c..3e2d90914 100644
--- a/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs
@@ -168,7 +168,7 @@ pub(super) fn read_norm_layer<B: Backend>(
let bias = if use_bias {
read_float_vector_exact::<B>(tree, "biases", channels, data_type)?
} else {
- context.create_array(&[channels], data_type, &format!("{label_prefix}_bias"))
+ context.create_array_zeros(&[channels], data_type, &format!("{label_prefix}_bias"))
};
Ok(StructuredAudioNorm {
scales,
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs
index f07975c82..0c2ec5054 100644
--- a/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs
@@ -148,7 +148,7 @@ impl StructuredAudioCodecGraph {
let mut lengths_array = resources.length_staging(batch_size);
lengths_array.as_slice_mut::<i32>()[..lengths_i32.len()].copy_from_slice(&lengths_i32);
- let output = context.create_array(
+ let output = context.create_array_zeros(
&[batch_size, frames, self.input_dim],
quantizer_resources.data_type,
"structured_audio_quantizer_output_nsc",
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs
index 494ab745b..9e3891f0f 100644
--- a/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs
@@ -88,7 +88,7 @@ impl StructuredAudioCodecGraph {
.collect::<AudioResult<Vec<_>>>()?;
let context = resources.context().clone();
return Ok(SubmittedDecodedPaddedAudio {
- output: context.create_array(&[0], DataType::F32, "structured_audio_empty_decode_output"),
+ output: context.create_array_zeros(&[0], DataType::F32, "structured_audio_empty_decode_output"),
channels: 1,
frames: out_lengths.iter().copied().max().unwrap_or(0),
lengths: out_lengths,
@@ -343,7 +343,7 @@ impl StructuredAudioCodecGraph {
x = tanh_enqueue(
&mut encoder,
&x,
- context.create_array(x.shape(), data_type, "structured_audio_tanh_output"),
+ context.create_array_zeros(x.shape(), data_type, "structured_audio_tanh_output"),
&kernels,
)?;
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
index a30f99533..65c336e98 100644
--- a/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
@@ -89,7 +89,7 @@ impl<B: Backend> DecodeWorkspace<B> {
} else {
"structured_audio_lengths_b"
};
- let array = context.create_array(&[min_elements], DataType::I32, label);
+ let array = context.create_array_zeros(&[min_elements], DataType::I32, label);
*slot = Some(array.clone());
array
}
@@ -200,7 +200,7 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {
return existing.clone();
}
}
- let array = self.context.create_array(&[min_elements], DataType::U32, "quantizer_token_staging");
+ let array = self.context.create_array_zeros(&[min_elements], DataType::U32, "quantizer_token_staging");
*slot = Some(array.clone());
array
}
@@ -215,7 +215,7 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {
return existing.clone();
}
}
- let array = self.context.create_array(&[min_elements], DataType::I32, "quantizer_length_staging");
+ let array = self.context.create_array_zeros(&[min_elements], DataType::I32, "quantizer_length_staging");
*slot = Some(array.clone());
array
}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 054db8d82..b89f34fb7 100644
--- a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -124,8 +124,8 @@ impl StructuredAudioCodecGraph {
max_sequence_length,
);
let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(
- context.clone(),
- decoder_config,
+ context.as_ref(),
+ &decoder_config,
&root_loader_view,
transformer_subtree_name,
);
@@ -200,17 +200,17 @@ impl StructuredAudioCodecGraph {
let residual_quantizers = self.config.n_codebooks;
let residual_count_for_shape = residual_quantizers.max(1);
let residual_codebook_rows_for_shape = self.codebook_size.max(1);
- let residual_codebooks = context.create_array(
+ let residual_codebooks = context.create_array_zeros(
&[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim],
data_type,
"structured_audio_quantizer_residual_codebooks",
);
- let residual_out_proj = context.create_array(
+ let residual_out_proj = context.create_array_zeros(
&[residual_count_for_shape, self.input_dim, codebook_dim],
data_type,
"structured_audio_quantizer_residual_out_proj",
);
- let residual_out_bias = context.create_array(
+ let residual_out_bias = context.create_array_zeros(
&[residual_count_for_shape, self.input_dim],
data_type,
"structured_audio_quantizer_residual_out_bias",
@@ -420,7 +420,7 @@ impl StructuredAudioCodecGraph {
return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);
}
- let output = context.create_array(
+ let output = context.create_array_zeros(
&[batch_size, frames, self.input_dim],
latent_nsc.data_type(),
"structured_audio_post_module_output_nsc",
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 0feb27b2c..6b9f0bd02 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -8,6 +8,7 @@ use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
DataType,
backends::common::{Backend, Encoder},
+ config::ModelMetadata,
encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
@@ -70,8 +71,11 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {
}
impl<B: Backend> Classifier<B> {
- pub fn new(model_path: &Path) -> Result<Self, Error> {
- let context = ClassifierContext::new(model_path)?;
+ pub fn new(
+ model_path: &Path,
+ model_metadata: &ModelMetadata,
+ ) -> Result<Self, Error> {
+ let context = ClassifierContext::new(model_path, model_metadata)?;
Ok(Self {
context,
})
diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rs
index dff68ae99..5ef7421b3 100644
--- a/crates/uzu/src/classifier/classifier_context.rs
+++ b/crates/uzu/src/classifier/classifier_context.rs
@@ -1,4 +1,4 @@
-use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
+use std::{cell::RefCell, fs::File, path::Path, rc::Rc};
use crate::{
DataType,
@@ -36,17 +36,12 @@ pub struct ClassifierContext<B: Backend> {
}
impl<B: Backend> ClassifierContext<B> {
- pub fn new(model_path: &Path) -> Result<Self, Error> {
+ pub fn new(
+ model_path: &Path,
+ model_metadata: &ModelMetadata,
+ ) -> Result<Self, Error> {
let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;
- let config_path = model_path.join("config.json");
- if !config_path.exists() {
- return Err(Error::UnableToLoadConfig);
- }
- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let model_metadata: ModelMetadata =
- serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
-
let classifier_model_config = model_metadata.model_config.as_classifier().ok_or(Error::UnableToLoadConfig)?;
let decoder_config =
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index 8ee3c352b..b97125177 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -33,14 +33,14 @@ pub struct Decoder<B: Backend> {
impl<B: Backend> Decoder<B> {
pub fn new(
- context: Rc<B::Context>,
- decoder_config: Rc<DecoderConfig>,
+ context: &B::Context,
+ decoder_config: &DecoderConfig,
root_weight_loader: &ParameterTree<B::Context>,
) -> Self {
let embedding_weight_loader = root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree");
let embed = Embedding::new(
- context.as_ref(),
+ context,
decoder_config.vocab_size as u32,
decoder_config.model_dim as u32,
&decoder_config.embedding_config,
@@ -61,8 +61,8 @@ impl<B: Backend> Decoder<B> {
/// Used by models whose token lookup weights and logits readout weights
#[cfg(all(feature = "audio-runtime", metal_backend))]
pub fn new_with_embedding_and_readout_subtrees(
- context: Rc<B::Context>,
- decoder_config: Rc<DecoderConfig>,
+ context: &B::Context,
+ decoder_config: &DecoderConfig,
root_weight_loader: &ParameterTree<B::Context>,
transformer_subtree: &str,
embedding_subtree: &str,
@@ -73,7 +73,7 @@ impl<B: Backend> Decoder<B> {
let readout_weight_loader = root_weight_loader.subtree(readout_subtree).expect("Failed to get readout subtree");
let embed = Embedding::new_with_lookup_and_readout_trees(
- context.as_ref(),
+ context,
decoder_config.vocab_size as u32,
decoder_config.model_dim as u32,
&decoder_config.embedding_config,
@@ -82,12 +82,8 @@ impl<B: Backend> Decoder<B> {
)
.expect("Failed to create embedding");
- let (layers, norm) = Self::build_transformer_layers_and_norm(
- context,
- decoder_config.clone(),
- root_weight_loader,
- transformer_subtree,
- );
+ let (layers, norm) =
+ Self::build_transformer_layers_and_norm(context, decoder_config, root_weight_loader, transformer_subtree);
Self {
embed,
@@ -97,8 +93,8 @@ impl<B: Backend> Decoder<B> {
}
pub(crate) fn build_transformer_layers_and_norm(
- context: Rc<B::Context>,
- decoder_config: Rc<DecoderConfig>,
+ context: &B::Context,
+ decoder_config: &DecoderConfig,
root_weight_loader: &ParameterTree<B::Context>,
transformer_subtree: &str,
) -> (Box<[LayerExecutables<B>]>, RMSNorm<B>) {
@@ -145,13 +141,13 @@ impl<B: Backend> Decoder<B> {
let layer_type = model_shape.layer_type(layer_index);
let rope_for_layer = match layer_type {
DecoderLayerType::Transformer => {
- let mut rope_block = global_rope.clone().expect("Global rope missing for transformer layer");
- if let (Some(_), Some(local_rope_block)) =
- (sliding_window_sizes[layer_index], local_rope.clone())
+ if let Some(_) = sliding_window_sizes[layer_index]
+ && let Some(local_rope_block) = local_rope.clone()
{
- rope_block = local_rope_block;
+ Some(local_rope_block)
+ } else {
+ Some(global_rope.clone().expect("Global rope missing for transformer layer"))
}
- Some(rope_block)
},
DecoderLayerType::StateSpace {
..
@@ -164,7 +160,7 @@ impl<B: Backend> Decoder<B> {
let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();
LayerExecutables::new(
- context.clone(),
+ context,
layer_config,
layer_type,
layer_index,
@@ -181,7 +177,7 @@ impl<B: Backend> Decoder<B> {
.collect::<Vec<_>>();
let norm_block = RMSNorm::new(
- context.as_ref(),
+ context,
norm_data_type,
decoder_config.output_norm_config.clone(),
ArrayId::Main,
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index 7c5ce2540..cde4dda7a 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -32,7 +32,7 @@ pub struct LayerExecutables<B: Backend> {
impl<B: Backend> LayerExecutables<B> {
#[allow(clippy::too_many_arguments)]
pub fn new(
- context: Rc<B::Context>,
+ context: &B::Context,
layer_config: &DecoderLayerConfig,
layer_type: &DecoderLayerType,
layer_index: usize,
@@ -45,21 +45,20 @@ impl<B: Backend> LayerExecutables<B> {
decoder_layer_loader: &ParameterTree<B::Context>,
rope: Option<Rc<Rope<B>>>,
) -> Self {
- let ctx = context.as_ref(); // Reference for functions expecting &B::Context
let intermediate_data_type: DataType = match &layer_config.mixer_config {
MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),
MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),
MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
};
let copy_main_to_shortcut = TensorCopy::<B>::new(
- ctx,
+ context,
intermediate_data_type,
vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
)
.unwrap();
let pre_attention_norm = RMSNorm::new(
- ctx,
+ context,
intermediate_data_type,
layer_config.pre_attention_norm_config.clone(),
ArrayId::Main,
@@ -77,7 +76,7 @@ impl<B: Backend> LayerExecutables<B> {
attention_config.has_qkv_biases,
model_dim,
[num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],
- ctx,
+ context,
&decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),
ArrayId::Main,
ArrayId::QKV,
@@ -91,7 +90,7 @@ impl<B: Backend> LayerExecutables<B> {
false,
model_dim,
[num_heads * head_dim],
- ctx,
+ context,
&decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),
ArrayId::Main,
ArrayId::Gate,
@@ -105,7 +104,7 @@ impl<B: Backend> LayerExecutables<B> {
let qk_norm =
if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
match QKNorm::new(
- ctx,
+ context,
intermediate_data_type,
attention_config.query_norm_config.clone(),
attention_config.key_norm_config.clone(),
@@ -127,7 +126,7 @@ impl<B: Backend> LayerExecutables<B> {
attention_config.has_out_biases,
num_heads * head_dim,
[model_dim],
- ctx,
+ context,
&decoder_layer_loader.subtree("mixer.out_projection").unwrap(),
ArrayId::AttentionOutput,
ArrayId::Main,
@@ -135,7 +134,7 @@ impl<B: Backend> LayerExecutables<B> {
.expect("Failed to create out projection");
let attention = Attention::new(
- ctx,
+ context,
intermediate_data_type,
layer_index,
attention_scale,
@@ -157,7 +156,7 @@ impl<B: Backend> LayerExecutables<B> {
},
MixerConfig::Mamba(mamba_config) => {
let mixer = MambaMixer::new(
- ctx,
+ context,
layer_type.clone(),
mamba_config.clone(),
layer_index,
@@ -173,7 +172,7 @@ impl<B: Backend> LayerExecutables<B> {
},
MixerConfig::ShortConv(short_conv_config) => {
let mixer = ShortConvMixer::new(
- ctx,
+ context,
layer_type.clone(),
short_conv_config.clone(),
layer_index,
@@ -189,7 +188,7 @@ impl<B: Backend> LayerExecutables<B> {
let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
Some(
RMSNorm::new(
- ctx,
+ context,
intermediate_data_type,
norm_config.clone(),
ArrayId::Main,
@@ -203,14 +202,14 @@ impl<B: Backend> LayerExecutables<B> {
};
let main_shortcut_add_swap = TensorAddSwap::<B>::new(
- ctx,
+ context,
intermediate_data_type,
vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
)
.unwrap();
let pre_mlp_norm = RMSNorm::new(
- ctx,
+ context,
intermediate_data_type,
layer_config.pre_mlp_norm_config.clone(),
ArrayId::Main,
@@ -223,7 +222,7 @@ impl<B: Backend> LayerExecutables<B> {
&layer_config.mlp_config,
model_dim,
hidden_dim,
- context.as_ref(),
+ context,
&decoder_layer_loader.subtree("mlp").unwrap(),
)
.expect("Failed to create mlp block");
@@ -231,7 +230,7 @@ impl<B: Backend> LayerExecutables<B> {
let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(
RMSNorm::new(
- ctx,
+ context,
intermediate_data_type,
norm_config.clone(),
ArrayId::Main,
diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rs
index 5fdbcdbd4..b000fe01b 100644
--- a/crates/uzu/src/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/forward_pass/cache_layers.rs
@@ -123,12 +123,12 @@ impl<B: Backend> CacheLayers<B> {
CacheLayer::Transformer(KVCacheLayer {
state: state.clone(),
- keys: context.create_array(
+ keys: context.create_array_zeros(
&shape,
model_shape.kv_cache_data_type(),
&format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),
),
- values: context.create_array(
+ values: context.create_array_zeros(
&shape,
model_shape.kv_cache_data_type(),
&format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),
@@ -158,12 +158,12 @@ impl<B: Backend> CacheLayers<B> {
let dtype = model_shape.activation_data_type();
CacheLayer::StateSpace(SSMLayer {
- conv_state: context.create_array(
+ conv_state: context.create_array_zeros(
&conv_shape,
dtype,
&format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),
),
- ssm_state: context.create_array(
+ ssm_state: context.create_array_zeros(
&ssm_shape,
dtype,
&format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),
@@ -179,12 +179,12 @@ impl<B: Backend> CacheLayers<B> {
let dtype = model_shape.activation_data_type();
CacheLayer::ShortConv(ShortConvLayer {
- conv_state: context.create_array(
+ conv_state: context.create_array_zeros(
&conv_shape,
dtype,
&format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),
),
- suffix_state: context.create_array(
+ suffix_state: context.create_array_zeros(
&suffix_state_shape,
dtype,
&format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),
@@ -374,12 +374,12 @@ impl<B: Backend> CacheLayers<B> {
}
let new_shape = [num_groups, new_total_len, head_dim];
- let mut new_keys = context.create_array(
+ let mut new_keys = context.create_array_zeros(
&new_shape,
dtype,
&format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),
);
- let mut new_values = context.create_array(
+ let mut new_values = context.create_array_zeros(
&new_shape,
dtype,
&format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),
@@ -401,7 +401,7 @@ impl<B: Backend> CacheLayers<B> {
CacheLayer::StateSpace(layer) => {
let conv_shape = layer.conv_state.shape().to_vec();
let conv_dtype = layer.conv_state.data_type();
- let mut new_conv = context.create_array(
+ let mut new_conv = context.create_array_uninitialized(
&conv_shape,
conv_dtype,
&format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),
@@ -410,7 +410,7 @@ impl<B: Backend> CacheLayers<B> {
let ssm_shape = layer.ssm_state.shape().to_vec();
let ssm_dtype = layer.ssm_state.data_type();
- let mut new_ssm = context.create_array(
+ let mut new_ssm = context.create_array_uninitialized(
&ssm_shape,
ssm_dtype,
&format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),
@@ -425,7 +425,7 @@ impl<B: Backend> CacheLayers<B> {
CacheLayer::ShortConv(layer) => {
let conv_shape = layer.conv_state.shape().to_vec();
let conv_dtype = layer.conv_state.data_type();
- let mut new_conv = context.create_array(
+ let mut new_conv = context.create_array_uninitialized(
&conv_shape,
conv_dtype,
&format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),
@@ -434,14 +434,11 @@ impl<B: Backend> CacheLayers<B> {
let suffix_shape = layer.suffix_state.shape().to_vec();
let suffix_dtype = layer.suffix_state.data_type();
- let mut new_suffix = context.create_array(
+ let new_suffix = context.create_array_zeros(
&suffix_shape,
suffix_dtype,
&format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),
);
- {
- new_suffix.as_bytes_mut().fill(0);
- }
CacheLayer::ShortConv(ShortConvLayer {
conv_state: new_conv,
diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rs
index 086c256e8..b31ca92c3 100644
--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs
@@ -325,8 +325,10 @@ impl<B: Backend> KVCacheLayer<B> {
let dtype = self.keys.data_type();
let slice_shape = [num_groups, len, head_dim];
- let mut slice_keys = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_keys");
- let mut slice_values = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_values");
+ let mut slice_keys =
+ context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_keys");
+ let mut slice_values =
+ context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_values");
let slots: Vec<usize> = (range.start..range.end)
.enumerate()
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 7e93aedf6..5ad622507 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -20,6 +20,7 @@ use crate::{
Backend, Buffer, CommandBuffer, Context, Encoder, Executable,
kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},
},
+ config::ModelMetadata,
encodable_block::EncodingParameters,
forward_pass::{
cache_layers::{CacheLayer, CacheLayersSlice},
@@ -689,10 +690,11 @@ impl<B: Backend> LanguageModelGenerator<B> {
pub fn new(
model_path: &Path,
decoding_config: DecodingConfig,
+ model_metadata: &ModelMetadata,
) -> Result<Self, Error> {
let gpu_capture = GpuCaptureManager::new();
- let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
+ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_metadata)?;
Ok(Self {
decoding_config,
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index b210b660d..1849f66d3 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -1,7 +1,6 @@
use std::{
cell::{Cell, RefCell},
fs::File,
- io::BufReader,
path::Path,
rc::Rc,
};
@@ -119,7 +118,7 @@ pub struct LanguageModelGeneratorContext<B: Backend> {
pub scratch_buffers: ScratchBuffers<B>,
pub model_config: LanguageModelConfig,
- pub decoder_config: Rc<DecoderConfig>,
+ pub decoder_config: DecoderConfig,
pub model_shape: ModelShape,
pub executables: Decoder<B>,
pub kv_cache_update: Box<KVCacheUpdate<B>>,
@@ -139,21 +138,14 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
pub fn new(
model_path: &Path,
decoding_config: &DecodingConfig,
+ model_metadata: &ModelMetadata,
) -> Result<Self, Error> {
let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;
- let config_path = model_path.join("config.json");
- if !config_path.exists() {
- return Err(Error::UnableToLoadConfig);
- }
- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
- let model_metadata: ModelMetadata =
- serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
-
// Extract language model config
let language_model_config = model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;
- let decoder_config = Rc::new(language_model_config.decoder_config().map_err(|_| Error::UnableToLoadConfig)?);
+ let decoder_config = language_model_config.decoder_config().map_err(|_| Error::UnableToLoadConfig)?;
let model_shape = ModelShape::from_decoder_config(&decoder_config);
let prefill_step_size = decoding_config.prefill_step_size.resolve(language_model_config);
@@ -175,7 +167,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let scratch_buffers =
ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_prefix_length, max_suffix_length);
- let executables = Decoder::new(context.clone(), decoder_config.clone(), &root_loader_view);
+ let executables = Decoder::new(context.as_ref(), &decoder_config, &root_loader_view);
let cache_layers = Rc::new(RefCell::new(CacheLayers::new(
context.as_ref(),
diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rs
index 529a8361e..cbe485766 100644
--- a/crates/uzu/src/parameters/loader.rs
+++ b/crates/uzu/src/parameters/loader.rs
@@ -113,16 +113,17 @@ where
let (offset, size) = (metadata_entry.offset, metadata_entry.size);
let array_key = key.replace(".", "_");
let array_label = format!("parameter_loader_{array_key}");
- let mut array = self.context.create_array(&metadata_entry.shape, metadata_entry.data_type, &array_label);
- let expected_size = array.size();
- if expected_size != size {
+ let mut array =
+ self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type, &array_label);
+ if array.size() != size {
return Err(ParameterLoaderError::SizeMismatch {
data_type: metadata_entry.data_type,
shape: metadata_entry.shape.to_owned(),
- expected_size,
+ expected_size: array.size(),
actual_size: size,
});
}
+
self.file.read_exact_at(array.as_bytes_mut(), offset as u64)?;
Ok(array)
}
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index d8031c18a..5854e5e2b 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -137,8 +137,10 @@ impl ChatSession {
let output_parser =
OutputParser::new(language_model_config.message_processor_config.output_parser_regex.clone())?;
- let llm =
- Box::new(LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone()).map_err(Error::from)?);
+ let llm = Box::new(
+ LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone(), &model_metadata)
+ .map_err(Error::from)?,
+ );
Ok(Self {
model_path,
diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rs
index cc3de24aa..f863ab7f4 100644
--- a/crates/uzu/src/session/classification_session.rs
+++ b/crates/uzu/src/session/classification_session.rs
@@ -47,7 +47,7 @@ impl ClassificationSession {
let input_processor =
Box::new(InputProcessorDefault::new(classifier_model_config.message_processor_config.clone()))
as Box<dyn InputProcessor>;
- let classifier = Box::new(Classifier::<B>::new(&model_path)?);
+ let classifier = Box::new(Classifier::<B>::new(&model_path, &model_metadata)?);
Ok(Self {
model_path,
diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rs
index a236b2207..3c0b43a59 100644
--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs
@@ -37,8 +37,8 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
let scratch_buffers =
ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_prefix_length, max_suffix_length);
let executables = Decoder::new_with_embedding_and_readout_subtrees(
- context.clone(),
- decoder_config.clone(),
+ context.as_ref(),
+ &decoder_config,
&root_loader_view,
transformer_subtree,
embedding_subtree,
@@ -236,9 +236,9 @@ impl<B: Backend> TokenDecoderRunner<B> {
<B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type)
.map_err(unable_to_create_context)?;
let single_hidden_capture =
- context.create_array(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");
+ context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");
let single_override_embedding =
- context.create_array(&[1, model_dim], activation_data_type, "tts_single_override_embedding");
+ context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_override_embedding");
Ok(Self {
ctx,
diff --git a/crates/uzu/src/session/tts_session/fishaudio.rs b/crates/uzu/src/session/tts_session/fishaudio.rs
index 52afd9d61..9ad5efdb2 100644
--- a/crates/uzu/src/session/tts_session/fishaudio.rs
+++ b/crates/uzu/src/session/tts_session/fishaudio.rs
@@ -109,7 +109,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {
let tensor_copy =
<B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;
let codebook_token_indices =
- context.create_array(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");
+ context.create_array_zeros(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");
Ok(Self {
embedding_rows_sum,
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index b134c9fa7..248b9cac8 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -185,7 +185,7 @@ impl<B: Backend> TraceValidator<B> {
serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
let context = if metadata.model_config.is_classifier() {
- let classifier = Classifier::new(model_path)?;
+ let classifier = Classifier::new(model_path, &metadata)?;
ModelContext::Classifier(classifier)
} else {
let prefill_step_size = Self::determine_prefill_step_size(model_path);
@@ -198,7 +198,7 @@ impl<B: Backend> TraceValidator<B> {
AsyncBatchSize::default(),
false,
);
- let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &metadata)?;
let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());
Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);
ModelContext::LanguageModelGenerator(llm_context)
diff --git a/crates/uzu/tests/audio_kernel_perf_test.rs b/crates/uzu/tests/audio_kernel_perf_test.rs
index 034bf919c..5a2653ef8 100644
--- a/crates/uzu/tests/audio_kernel_perf_test.rs
+++ b/crates/uzu/tests/audio_kernel_perf_test.rs
@@ -121,17 +121,21 @@ fn audio_kernel_perf() {
let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();
- let tokens = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");
+ let tokens =
+ context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");
let lengths = make_lengths(&context, batch_size, frames as i32);
- let semantic_codebook = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");
- let semantic_out_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");
- let semantic_out_bias = context.create_array(&[INPUT_DIM], dt, "perf_sem_bias");
- let residual_codebooks =
- context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "perf_res_cbs");
+ let semantic_codebook = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");
+ let semantic_out_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");
+ let semantic_out_bias = context.create_array_zeros(&[INPUT_DIM], dt, "perf_sem_bias");
+ let residual_codebooks = context.create_array_zeros(
+ &[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM],
+ dt,
+ "perf_res_cbs",
+ );
let residual_out_proj =
- context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");
- let residual_out_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");
- let output = context.create_array(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");
+ context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");
+ let residual_out_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");
+ let output = context.create_array_zeros(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");
let label = format!(
"QuantizerDecode [B={}, T={}, K={}, dim={}, cdim={}]",
@@ -204,10 +208,10 @@ fn audio_kernel_perf() {
let seq_in = frames;
let seq_out = seq_in * stride;
let ksize = stride * 2;
- let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");
- let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");
- let bias = context.create_array(&[cout], dt, "tconv_b");
- let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");
+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");
+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");
+ let bias = context.create_array_zeros(&[cout], dt, "tconv_b");
+ let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");
let lengths = make_lengths(&context, batch_size, seq_out as i32);
let label = format!("TransConv {tag} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");
@@ -257,10 +261,10 @@ fn audio_kernel_perf() {
let seq_in = frames;
let seq_out = seq_in * stride;
let ksize = stride * 2;
- let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");
- let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");
- let bias = context.create_array(&[cout], dt, "tconv_b");
- let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");
+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");
+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");
+ let bias = context.create_array_zeros(&[cout], dt, "tconv_b");
+ let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");
let lengths = make_lengths(&context, batch_size, seq_out as i32);
let label = format!("TransConv dec{block_idx} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");
@@ -317,10 +321,10 @@ fn audio_kernel_perf() {
let cin = UP_CHANNELS[2]; // 256
let cout = DECODER_DIM; // 1536
let ksize = 7;
- let input = context.create_array(&[batch_size * cin * fc_seq], dt, "fc_in");
- let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");
- let bias = context.create_array(&[cout], dt, "fc_b");
- let output = context.create_array(&[batch_size * cout * fc_seq], dt, "fc_out");
+ let input = context.create_array_zeros(&[batch_size * cin * fc_seq], dt, "fc_in");
+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");
+ let bias = context.create_array_zeros(&[cout], dt, "fc_b");
+ let output = context.create_array_zeros(&[batch_size * cout * fc_seq], dt, "fc_out");
let lengths = make_lengths(&context, batch_size, fc_seq as i32);
let label = format!("CausalConv1d first_conv [{cin}->{cout}, k={ksize}, T={fc_seq}]");
@@ -368,10 +372,10 @@ fn audio_kernel_perf() {
for &dilation in &[1, 3, 9] {
let label =
format!("CausalConv1d res dec{block_idx} [{ch}->{ch}, k={ksize}, d={dilation}, T={dec_frames}]");
- let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");
- let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");
- let bias = context.create_array(&[ch], dt, "res_b");
- let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");
+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");
+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");
+ let bias = context.create_array_zeros(&[ch], dt, "res_b");
+ let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");
let lengths = make_lengths(&context, batch_size, dec_frames as i32);
let ms = measure(&label, || {
@@ -415,10 +419,10 @@ fn audio_kernel_perf() {
let cout = 1;
let ksize = 7;
let final_frames = SEMANTIC_FRAMES_10S * 2 * 2 * 8 * 8 * 4 * 2; // full resolution
- let input = context.create_array(&[batch_size * cin * final_frames], dt, "fc_in");
- let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");
- let bias = context.create_array(&[cout], dt, "fc_b");
- let output = context.create_array(&[batch_size * cout * final_frames], dt, "fc_out");
+ let input = context.create_array_zeros(&[batch_size * cin * final_frames], dt, "fc_in");
+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");
+ let bias = context.create_array_zeros(&[cout], dt, "fc_b");
+ let output = context.create_array_zeros(&[batch_size * cout * final_frames], dt, "fc_out");
let lengths = make_lengths(&context, batch_size, final_frames as i32);
let label = format!("CausalConv1d final_conv [{cin}->{cout}, k={ksize}, T={final_frames}]");
@@ -473,11 +477,11 @@ fn audio_kernel_perf() {
let ksize = 7;
let label = format!("CausalConv1dResidual dec{block_idx} [{ch}->{ch}, k={ksize}, T={dec_frames}]");
- let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");
- let residual = context.create_array(&[batch_size * ch * dec_frames], dt, "res_res");
- let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");
- let bias = context.create_array(&[ch], dt, "res_b");
- let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");
+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");
+ let residual = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_res");
+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");
+ let bias = context.create_array_zeros(&[ch], dt, "res_b");
+ let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");
let lengths = make_lengths(&context, batch_size, dec_frames as i32);
let ms = measure(&label, || {
@@ -532,10 +536,10 @@ fn audio_kernel_perf() {
let ksize = 7;
let label = format!("CausalConv1dGrouped dw up{block_idx} [{ch}, g={ch}, k={ksize}, T={frames}]");
- let input = context.create_array(&[batch_size * ch * frames], dt, "dw_in");
- let weight = context.create_array(&[ch * 1 * ksize], dt, "dw_w");
- let bias = context.create_array(&[ch], dt, "dw_b");
- let output = context.create_array(&[batch_size * ch * frames], dt, "dw_out");
+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_in");
+ let weight = context.create_array_zeros(&[ch * 1 * ksize], dt, "dw_w");
+ let bias = context.create_array_zeros(&[ch], dt, "dw_b");
+ let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_out");
let lengths = make_lengths(&context, batch_size, frames as i32);
let ms = measure(&label, || {
@@ -588,9 +592,9 @@ fn audio_kernel_perf() {
let ch_in = DEC_CHANNELS[block_idx];
{
let label = format!("HalfSnake pre-tconv dec{block_idx} [{ch_in}ch, T={dec_frames}]");
- let input = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_in");
- let alpha = context.create_array(&[ch_in], dt, "sn_a");
- let output = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_out");
+ let input = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_in");
+ let alpha = context.create_array_zeros(&[ch_in], dt, "sn_a");
+ let output = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_out");
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
@@ -624,9 +628,9 @@ fn audio_kernel_perf() {
let ch_out = DEC_CHANNELS[block_idx + 1];
{
let label = format!("HalfSnake res dec{block_idx} [{ch_out}ch, T={dec_frames}]");
- let input = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_in");
- let alpha = context.create_array(&[ch_out], dt, "sn_a");
- let output = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_out");
+ let input = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_in");
+ let alpha = context.create_array_zeros(&[ch_out], dt, "sn_a");
+ let output = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_out");
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
@@ -673,10 +677,10 @@ fn audio_kernel_perf() {
// Norm
{
let label = format!("NormNcs up{block_idx} [{ch}ch, T={frames}]");
- let input = context.create_array(&[batch_size * ch * frames], dt, "norm_in");
- let scales = context.create_array(&[ch], dt, "norm_s");
- let bias = context.create_array(&[ch], dt, "norm_b");
- let output = context.create_array(&[batch_size * ch * frames], dt, "norm_out");
+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_in");
+ let scales = context.create_array_zeros(&[ch], dt, "norm_s");
+ let bias = context.create_array_zeros(&[ch], dt, "norm_b");
+ let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_out");
let lengths = make_lengths(&context, batch_size, frames as i32);
let ms = measure(&label, || {
@@ -714,10 +718,10 @@ fn audio_kernel_perf() {
// Pointwise conv1 (ch -> ch)
{
let label = format!("Conv1d pw up{block_idx} [{ch}->{ch}, k=1, T={frames}]");
- let input = context.create_array(&[batch_size * ch * frames], dt, "pw_in");
- let weight = context.create_array(&[ch * ch], dt, "pw_w");
- let bias = context.create_array(&[ch], dt, "pw_b");
- let output = context.create_array(&[batch_size * ch * frames], dt, "pw_out");
+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_in");
+ let weight = context.create_array_zeros(&[ch * ch], dt, "pw_w");
+ let bias = context.create_array_zeros(&[ch], dt, "pw_b");
+ let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_out");
let lengths = make_lengths(&context, batch_size, frames as i32);
let ms = measure(&label, || {
@@ -761,8 +765,8 @@ fn audio_kernel_perf() {
{
let n = ch * frames;
let label = format!("GELU up{block_idx} [n={n}]");
- let input = context.create_array(&[n], dt, "gelu_in");
- let output = context.create_array(&[n], dt, "gelu_out");
+ let input = context.create_array_zeros(&[n], dt, "gelu_in");
+ let output = context.create_array_zeros(&[n], dt, "gelu_out");
let ms = measure(&label, || {
let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
@@ -809,15 +813,15 @@ fn audio_kernel_perf() {
let frames = SEMANTIC_FRAMES_10S;
// Quantizer arrays
- let p_q_tok = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");
+ let p_q_tok = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");
let p_q_len = make_lengths(&context, batch_size, frames as i32);
- let p_q_sem_cb = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");
- let p_q_sem_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");
- let p_q_sem_bias = context.create_array(&[INPUT_DIM], dt, "p_sem_bias");
+ let p_q_sem_cb = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");
+ let p_q_sem_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");
+ let p_q_sem_bias = context.create_array_zeros(&[INPUT_DIM], dt, "p_sem_bias");
let p_q_res_cbs =
- context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");
- let p_q_res_proj = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");
- let p_q_res_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");
+ context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");
+ let p_q_res_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");
+ let p_q_res_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");
// Compute frame counts at each stage
let f_up0 = frames * UP_STRIDES[0]; // 430
@@ -830,42 +834,42 @@ fn audio_kernel_perf() {
// Pipeline scratch: large enough for the biggest intermediate
// Biggest is decoder_dim * f_up1 or ch * f_dec stages
let max_elems: usize = batch_size * DECODER_DIM * f_dec3.max(f_up1);
- let scratch_a = context.create_array(&[max_elems], dt, "pipe_sa");
- let scratch_b = context.create_array(&[max_elems], dt, "pipe_sb");
+ let scratch_a = context.create_array_zeros(&[max_elems], dt, "pipe_sa");
+ let scratch_b = context.create_array_zeros(&[max_elems], dt, "pipe_sb");
// Third scratch buffer for residual connections (avoids RefCell borrow conflict)
- let scratch_r = context.create_array(&[max_elems], dt, "pipe_sr");
+ let scratch_r = context.create_array_zeros(&[max_elems], dt, "pipe_sr");
// Upsample block 0: 1024->512, stride=2
- let u0_w = context.create_array(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");
- let u0_b = context.create_array(&[UP_CHANNELS[1]], dt, "u0_b");
+ let u0_w = context.create_array_zeros(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");
+ let u0_b = context.create_array_zeros(&[UP_CHANNELS[1]], dt, "u0_b");
// ConvNeXt 0 (ch=512)
let c0_ch = UP_CHANNELS[1];
- let c0_dw_w = context.create_array(&[c0_ch * 1 * 7], dt, "c0_dw_w");
- let c0_dw_b = context.create_array(&[c0_ch], dt, "c0_dw_b");
- let c0_ns = context.create_array(&[c0_ch], dt, "c0_ns");
- let c0_nb = context.create_array(&[c0_ch], dt, "c0_nb");
- let c0_p1_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p1_w");
- let c0_p1_b = context.create_array(&[c0_ch], dt, "c0_p1_b");
- let c0_p2_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p2_w");
- let c0_p2_b = context.create_array(&[c0_ch], dt, "c0_p2_b");
+ let c0_dw_w = context.create_array_zeros(&[c0_ch * 1 * 7], dt, "c0_dw_w");
+ let c0_dw_b = context.create_array_zeros(&[c0_ch], dt, "c0_dw_b");
+ let c0_ns = context.create_array_zeros(&[c0_ch], dt, "c0_ns");
+ let c0_nb = context.create_array_zeros(&[c0_ch], dt, "c0_nb");
+ let c0_p1_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p1_w");
+ let c0_p1_b = context.create_array_zeros(&[c0_ch], dt, "c0_p1_b");
+ let c0_p2_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p2_w");
+ let c0_p2_b = context.create_array_zeros(&[c0_ch], dt, "c0_p2_b");
// Upsample block 1: 512->256, stride=2
- let u1_w = context.create_array(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");
- let u1_b = context.create_array(&[UP_CHANNELS[2]], dt, "u1_b");
+ let u1_w = context.create_array_zeros(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");
+ let u1_b = context.create_array_zeros(&[UP_CHANNELS[2]], dt, "u1_b");
// ConvNeXt 1 (ch=256)
let c1_ch = UP_CHANNELS[2];
- let c1_dw_w = context.create_array(&[c1_ch * 1 * 7], dt, "c1_dw_w");
- let c1_dw_b = context.create_array(&[c1_ch], dt, "c1_dw_b");
- let c1_ns = context.create_array(&[c1_ch], dt, "c1_ns");
- let c1_nb = context.create_array(&[c1_ch], dt, "c1_nb");
- let c1_p1_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p1_w");
- let c1_p1_b = context.create_array(&[c1_ch], dt, "c1_p1_b");
- let c1_p2_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p2_w");
- let c1_p2_b = context.create_array(&[c1_ch], dt, "c1_p2_b");
+ let c1_dw_w = context.create_array_zeros(&[c1_ch * 1 * 7], dt, "c1_dw_w");
+ let c1_dw_b = context.create_array_zeros(&[c1_ch], dt, "c1_dw_b");
+ let c1_ns = context.create_array_zeros(&[c1_ch], dt, "c1_ns");
+ let c1_nb = context.create_array_zeros(&[c1_ch], dt, "c1_nb");
+ let c1_p1_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p1_w");
+ let c1_p1_b = context.create_array_zeros(&[c1_ch], dt, "c1_p1_b");
+ let c1_p2_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p2_w");
+ let c1_p2_b = context.create_array_zeros(&[c1_ch], dt, "c1_p2_b");
// first_conv: 256->1536, k=7
- let fc_w = context.create_array(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");
- let fc_b = context.create_array(&[DECODER_DIM], dt, "fc_b");
+ let fc_w = context.create_array_zeros(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");
+ let fc_b = context.create_array_zeros(&[DECODER_DIM], dt, "fc_b");
// Decoder block weights for all 4 blocks
struct DecBlockWeights {
@@ -888,27 +892,27 @@ fn audio_kernel_perf() {
let stride = DEC_STRIDES[i];
let ksize = stride * 2;
DecBlockWeights {
- snake_a: context.create_array(&[cin], dt, &format!("d{i}_sa")),
- tconv_w: context.create_array(&[cin * cout * ksize], dt, &format!("d{i}_tw")),
- tconv_b: context.create_array(&[cout], dt, &format!("d{i}_tb")),
- ru_s1: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s1"))),
+ snake_a: context.create_array_zeros(&[cin], dt, &format!("d{i}_sa")),
+ tconv_w: context.create_array_zeros(&[cin * cout * ksize], dt, &format!("d{i}_tw")),
+ tconv_b: context.create_array_zeros(&[cout], dt, &format!("d{i}_tb")),
+ ru_s1: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s1"))),
ru_c1_w: std::array::from_fn(|j| {
- context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))
+ context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))
}),
- ru_c1_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c1b"))),
- ru_s2: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s2"))),
+ ru_c1_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c1b"))),
+ ru_s2: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s2"))),
ru_c2_w: std::array::from_fn(|j| {
- context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))
+ context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))
}),
- ru_c2_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c2b"))),
+ ru_c2_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c2b"))),
}
})
.collect();
// final snake + conv + tanh
- let final_snake_a = context.create_array(&[DEC_CHANNELS[4]], dt, "final_sa");
- let final_conv_w = context.create_array(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");
- let final_conv_b = context.create_array(&[1], dt, "final_cb");
+ let final_snake_a = context.create_array_zeros(&[DEC_CHANNELS[4]], dt, "final_sa");
+ let final_conv_w = context.create_array_zeros(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");
+ let final_conv_b = context.create_array_zeros(&[1], dt, "final_cb");
// Length arrays
let len_up0 = make_lengths(&context, batch_size, f_up0 as i32);
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index ac66869ed..42407e933 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -3,6 +3,8 @@
pub mod assert;
pub mod audio_nanocodec_fsq_reference;
pub mod audio_nanocodec_ops_reference;
+pub mod path;
+pub mod perf;
pub mod proptest;
/// Invokes `$body` once per available backend, with `$B` bound to each backend type.
diff --git a/crates/uzu/tests/integration/util/path.rs b/crates/uzu/tests/common/path.rs
similarity index 100%
rename from crates/uzu/tests/integration/util/path.rs
rename to crates/uzu/tests/common/path.rs
diff --git a/crates/uzu/tests/common/perf.rs b/crates/uzu/tests/common/perf.rs
new file mode 100644
index 000000000..bf4db4c8c
--- /dev/null
+++ b/crates/uzu/tests/common/perf.rs
@@ -0,0 +1,75 @@
+use std::time::Instant;
+
+pub struct PerfResult {
+ pub name: String,
+ pub mean_ms: f64,
+ pub median_ms: f64,
+ pub min_ms: f64,
+ pub max_ms: f64,
+ pub std_dev_ms: f64,
+}
+
+impl PerfResult {
+ fn new(
+ name: String,
+ times_ms: &[f64],
+ ) -> Self {
+ let mut sorted = times_ms.to_vec();
+ sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());
+
+ let mean = sorted.iter().sum::<f64>() / sorted.len() as f64;
+ let median = sorted[sorted.len() / 2];
+ let min = sorted[0];
+ let max = sorted[sorted.len() - 1];
+
+ let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / sorted.len() as f64;
+ let std_dev = variance.sqrt();
+
+ Self {
+ name,
+ mean_ms: mean,
+ median_ms: median,
+ min_ms: min,
+ max_ms: max,
+ std_dev_ms: std_dev,
+ }
+ }
+
+ pub fn print(&self) {
+ eprintln!(
+ " {:<30} mean={:8.3}ms median={:8.3}ms min={:8.3}ms max={:8.3}ms std={:6.3}ms",
+ self.name, self.mean_ms, self.median_ms, self.min_ms, self.max_ms, self.std_dev_ms
+ );
+ }
+}
+
+pub fn run_perf<F: FnMut()>(
+ name: &str,
+ iterations: usize,
+ f: F,
+) -> PerfResult {
+ run_perf_with_warmup(name, 0, iterations, f)
+}
+
+pub fn run_perf_with_warmup<F: FnMut()>(
+ name: &str,
+ warmup: usize,
+ iterations: usize,
+ mut f: F,
+) -> PerfResult {
+ // Warmup
+ for _ in 0..warmup {
+ f();
+ }
+
+ // Measure
+ let mut times_ms = Vec::new();
+ for _ in 0..iterations {
+ let start = Instant::now();
+ f();
+ let elapsed = start.elapsed();
+ times_ms.push(elapsed.as_secs_f64() * 1000.0);
+ }
+
+ PerfResult::new(name.to_string(), &times_ms)
+}
diff --git a/crates/uzu/tests/integration/parameters/loader_test.rs b/crates/uzu/tests/integration/parameters/loader_test.rs
index a717f1a3b..a2988ba36 100644
--- a/crates/uzu/tests/integration/parameters/loader_test.rs
+++ b/crates/uzu/tests/integration/parameters/loader_test.rs
@@ -12,7 +12,7 @@ use uzu::{
},
};
-use crate::util::path::get_test_weights_path;
+use crate::common::path::get_test_weights_path;
#[tag(heavy)]
#[test]
diff --git a/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rs b/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rs
index 23a4ef027..b7b85bf48 100644
--- a/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rs
+++ b/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rs
@@ -2,7 +2,7 @@ use std::fs::File;
use uzu::read_safetensors_metadata;
-use crate::util::path::get_test_weights_path;
+use crate::common::path::get_test_weights_path;
#[test]
fn test_metadata_loading() {
diff --git a/crates/uzu/tests/integration/session/chat_session/context_mode_test.rs b/crates/uzu/tests/integration/session/chat_session/context_mode_test.rs
index af31c1f20..5ae387c0e 100644
--- a/crates/uzu/tests/integration/session/chat_session/context_mode_test.rs
+++ b/crates/uzu/tests/integration/session/chat_session/context_mode_test.rs
@@ -12,7 +12,7 @@ use uzu::session::{
types::{Input, Message, Output},
};
-use crate::util::path::get_test_model_path;
+use crate::common::path::get_test_model_path;
fn build_model_path() -> PathBuf {
get_test_model_path()
diff --git a/crates/uzu/tests/integration/session/chat_session/text_session_test.rs b/crates/uzu/tests/integration/session/chat_session/text_session_test.rs
index 97f56b578..180706a65 100644
--- a/crates/uzu/tests/integration/session/chat_session/text_session_test.rs
+++ b/crates/uzu/tests/integration/session/chat_session/text_session_test.rs
@@ -8,7 +8,7 @@ use uzu::session::{
types::{Input, Message, Output},
};
-use crate::util::path::get_test_model_path;
+use crate::common::path::get_test_model_path;
fn build_decoding_config() -> DecodingConfig {
DecodingConfig::default().with_sampling_seed(SamplingSeed::Custom(42))
diff --git a/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rs b/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rs
index f8c4095e9..b9f247224 100644
--- a/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rs
+++ b/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rs
@@ -12,7 +12,7 @@ use uzu::session::{
types::Input,
};
-use crate::util::{path::get_test_model_path, speculator::RepeatSpeculator};
+use crate::{common::path::get_test_model_path, util::speculator::RepeatSpeculator};
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
struct Address {
diff --git a/crates/uzu/tests/integration/tracer/trace_validator_test.rs b/crates/uzu/tests/integration/tracer/trace_validator_test.rs
index a98bf2cdc..477e46063 100644
--- a/crates/uzu/tests/integration/tracer/trace_validator_test.rs
+++ b/crates/uzu/tests/integration/tracer/trace_validator_test.rs
@@ -2,7 +2,7 @@
use uzu::{TraceValidator, backends::metal::Metal};
-use crate::util::path::{get_test_model_path, get_traces_path};
+use crate::common::path::{get_test_model_path, get_traces_path};
#[test]
fn test_tracer() {
diff --git a/crates/uzu/tests/integration/util/mod.rs b/crates/uzu/tests/integration/util/mod.rs
index a20a93af9..c91484f20 100644
--- a/crates/uzu/tests/integration/util/mod.rs
+++ b/crates/uzu/tests/integration/util/mod.rs
@@ -1,4 +1,2 @@
-#[cfg_attr(not(feature = "tracing"), allow(dead_code))]
-pub mod path;
#[cfg(metal_backend)]
pub mod speculator;
diff --git a/crates/uzu/tests/performance/mod.rs b/crates/uzu/tests/performance/mod.rs
index 01ae6f730..9e2de76d1 100644
--- a/crates/uzu/tests/performance/mod.rs
+++ b/crates/uzu/tests/performance/mod.rs
@@ -1,2 +1,7 @@
+#[macro_use]
+#[path = "../common/mod.rs"]
+mod common;
+
#[cfg(metal_backend)]
mod matmul;
+mod model_loading_perf_test;
diff --git a/crates/uzu/tests/performance/model_loading_perf_test.rs b/crates/uzu/tests/performance/model_loading_perf_test.rs
new file mode 100644
index 000000000..fc1645955
--- /dev/null
+++ b/crates/uzu/tests/performance/model_loading_perf_test.rs
@@ -0,0 +1,13 @@
+use uzu::prelude::{ChatSession, DecodingConfig};
+
+use crate::common::{path::get_test_model_path, perf::run_perf};
+
+#[test]
+#[ignore]
+fn test_perf_model_loading() {
+ let perf_results = run_perf("Model loading", 10, || {
+ let model_path = get_test_model_path();
+ let _ = ChatSession::new(model_path, DecodingConfig::default());
+ });
+ perf_results.print();
+}
diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
index b52980d71..eb4f77a06 100644
--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
@@ -45,8 +45,8 @@ fn make_test_layer(
};
let shape = [1, total_len.max(1), 1];
- let keys = context.create_array(&shape, DataType::F32, "kv_cache_keys");
- let values = context.create_array(&shape, DataType::F32, "kv_cache_values");
+ let keys = context.create_array_zeros(&shape, DataType::F32, "kv_cache_keys");
+ let values = context.create_array_zeros(&shape, DataType::F32, "kv_cache_values");
let prefix_token_positions = match &state {
KVCacheLayerState::Full {
diff --git a/crates/uzu/tests/unit/kernel/audio/audio_test.rs b/crates/uzu/tests/unit/kernel/audio/audio_test.rs
index fc876cf92..e53ead660 100644
--- a/crates/uzu/tests/unit/kernel/audio/audio_test.rs
+++ b/crates/uzu/tests/unit/kernel/audio/audio_test.rs
@@ -80,19 +80,19 @@ fn audio_conv1d_replicate_matches_reference_f32() {
let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07 - 1.3).cos() * 0.2).collect();
let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.01 - 0.02).collect();
- let mut input = context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");
+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");
+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array(&[cout], DataType::F32, "audio_conv1d_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_conv1d_bias");
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_conv1d_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_conv1d_lengths");
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
- let output = context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");
+ let output = context.create_array_zeros(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");
let expected = conv1d_reference(Conv1dSpec {
input: &input_values,
weight: &weight_values,
@@ -173,19 +173,19 @@ fn audio_causal_conv1d_matches_reference_f32() {
let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.02 - 0.5).cos() * 0.1).collect();
let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.001 - 0.002).collect();
- let mut input = context.create_array(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");
+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");
+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv1d_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv1d_bias");
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");
+ let mut lengths_array = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");
lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);
- let output = context.create_array(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");
+ let output = context.create_array_zeros(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");
let expected = causal_conv1d_reference(CausalConv1dSpec {
input: &input_values,
weight: &weight_values,
@@ -263,21 +263,27 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
let bias_values: Vec<f32> = vec![0.01, -0.02];
let mut input =
- context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");
+ context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut weight =
- context.create_array(&[cout, cin / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");
+ let mut weight = context.create_array_zeros(
+ &[cout, cin / groups, kernel_size],
+ DataType::F32,
+ "audio_causal_conv_transpose_weight",
+ );
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
- let output =
- context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_causal_conv_transpose_output");
+ let output = context.create_array_zeros(
+ &[batch_size, cout, seq_len_out],
+ DataType::F32,
+ "audio_causal_conv_transpose_output",
+ );
let expected = causal_conv_transpose1d_reference(CausalConvTranspose1dSpec {
input: &input_values,
weight: &weight_values,
@@ -385,24 +391,25 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {
("NCS", 0_i32, [batch_size, cin, seq_len_in], input_values_ncs.as_slice()),
("NSC", 1_i32, [batch_size, seq_len_in, cin], input_values_nsc.as_slice()),
] {
- let mut input = context.create_array(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");
+ let mut input =
+ context.create_array_zeros(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");
input.as_slice_mut::<f32>().copy_from_slice(input_values);
- let mut weight = context.create_array(
+ let mut weight = context.create_array_zeros(
&[cout, cin / groups, kernel_size],
DataType::F32,
"audio_causal_conv_transpose_lalamo_weight",
);
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");
+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
let mut lengths =
- context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");
+ context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
- let output = context.create_array(
+ let output = context.create_array_zeros(
&[batch_size, cout, seq_len_out],
DataType::F32,
"audio_causal_conv_transpose_lalamo_output",
@@ -457,11 +464,11 @@ fn audio_leaky_relu_matches_reference_f32() {
let eps = 1e-6f32;
let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
- let mut input = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");
+ let mut input = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let alpha = context.create_array(&[1], DataType::F32, "audio_leaky_relu_alpha");
- let output = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");
+ let alpha = context.create_array_zeros(&[1], DataType::F32, "audio_leaky_relu_alpha");
+ let output = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
@@ -502,10 +509,10 @@ fn audio_tanh_matches_reference_f32() {
let n = 1024usize;
let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
- let mut input = context.create_array(&[n], DataType::F32, "audio_tanh_input");
+ let mut input = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");
+ let output = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_output");
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
@@ -531,13 +538,13 @@ fn audio_add_matches_reference_f32() {
let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();
let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();
- let mut a = context.create_array(&[n], DataType::F32, "audio_add_a");
+ let mut a = context.create_array_zeros(&[n], DataType::F32, "audio_add_a");
a.as_slice_mut::<f32>().copy_from_slice(&a_values);
- let mut b = context.create_array(&[n], DataType::F32, "audio_add_b");
+ let mut b = context.create_array_zeros(&[n], DataType::F32, "audio_add_b");
b.as_slice_mut::<f32>().copy_from_slice(&b_values);
- let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");
+ let sum = context.create_array_zeros(&[n], DataType::F32, "audio_add_sum");
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(a_buffer = a);
@@ -578,13 +585,14 @@ fn audio_half_snake_matches_reference_f32() {
let input_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.01 - 3.2).sin()).collect();
let alpha_values: Vec<f32> = (0..snake_channels).map(|i| 0.5 + i as f32 * 0.1).collect();
- let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");
+ let mut input =
+ context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut alpha = context.create_array(&[snake_channels], DataType::F32, "audio_half_snake_alpha");
+ let mut alpha = context.create_array_zeros(&[snake_channels], DataType::F32, "audio_half_snake_alpha");
alpha.as_slice_mut::<f32>().copy_from_slice(&alpha_values);
- let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");
+ let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
@@ -683,19 +691,19 @@ fn audio_norm_ncs_matches_reference_f32() {
};
for subtract_mean in [false, true] {
- let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");
+ let mut input = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut scales = context.create_array(&[channels], DataType::F32, "audio_norm_scales");
+ let mut scales = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_scales");
scales.as_slice_mut::<f32>().copy_from_slice(&scales_values);
- let mut bias = context.create_array(&[channels], DataType::F32, "audio_norm_bias");
+ let mut bias = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_bias");
bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_norm_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_norm_lengths");
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);
- let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");
+ let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
@@ -749,17 +757,20 @@ fn audio_fsq_decode_matches_reference() {
let num_levels: Vec<i32> = vec![8, 5];
let dim_base_index: Vec<i32> = vec![1, 8];
- let mut tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");
+ let mut tokens = context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");
tokens.as_slice_mut::<i32>().copy_from_slice(&[
0, 7, 11, // g0 tokens at t0..t2
3, 5, 9, // g1 tokens at t0..t2
]);
- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_lengths");
lengths.as_slice_mut::<i32>().copy_from_slice(&[2]);
- let output =
- context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");
+ let output = context.create_array_zeros(
+ &[batch_size, num_groups * codebook_dim, seq_len],
+ DataType::F32,
+ "audio_fsq_output",
+ );
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(tokens_buffer = tokens);
@@ -812,17 +823,18 @@ fn audio_fsq_encode_matches_reference() {
];
let lengths_values = vec![3_i32];
- let mut input = context.create_array(
+ let mut input = context.create_array_zeros(
&[batch_size, num_groups * codebook_dim, seq_len],
DataType::F32,
"audio_fsq_encode_input",
);
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");
+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);
- let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");
+ let tokens =
+ context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");
run_command_buffer(&context, |command_buffer| {
borrow_array_buffer!(input_buffer = input);
diff --git a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs
index 01a5f181f..c64580a5b 100644
--- a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs
@@ -1,7 +1,5 @@
#![cfg(metal_backend)]
-use std::time::Instant;
-
use half::bf16;
use rand::{RngExt, SeedableRng, rngs::StdRng};
use uzu::{
@@ -20,75 +18,7 @@ use uzu::{
};
use super::test_utils::{alloc_buffer, alloc_buffer_with_data, create_ctx};
-
-struct PerfResult {
- name: String,
- mean_ms: f64,
- median_ms: f64,
- min_ms: f64,
- max_ms: f64,
- std_dev_ms: f64,
-}
-
-impl PerfResult {
- fn new(
- name: String,
- times_ms: &[f64],
- ) -> Self {
- let mut sorted = times_ms.to_vec();
- sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());
-
- let mean = sorted.iter().sum::<f64>() / sorted.len() as f64;
- let median = sorted[sorted.len() / 2];
- let min = sorted[0];
- let max = sorted[sorted.len() - 1];
-
- let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / sorted.len() as f64;
- let std_dev = variance.sqrt();
-
- Self {
- name,
- mean_ms: mean,
- median_ms: median,
- min_ms: min,
- max_ms: max,
- std_dev_ms: std_dev,
- }
- }
-
- fn print(&self) {
- eprintln!(
- " {:<30} mean={:8.3}ms median={:8.3}ms min={:8.3}ms max={:8.3}ms std={:6.3}ms",
- self.name, self.mean_ms, self.median_ms, self.min_ms, self.max_ms, self.std_dev_ms
- );
- }
-}
-
-fn time_kernel<F>(
- name: &str,
- warmup: usize,
- iterations: usize,
- mut f: F,
-) -> PerfResult
-where
- F: FnMut(),
-{
- // Warmup
- for _ in 0..warmup {
- f();
- }
-
- // Measure
- let mut times_ms = Vec::new();
- for _ in 0..iterations {
- let start = Instant::now();
- f();
- let elapsed = start.elapsed();
- times_ms.push(elapsed.as_secs_f64() * 1000.0);
- }
-
- PerfResult::new(name.to_string(), &times_ms)
-}
+use crate::common::perf::run_perf_with_warmup;
// Test E2E MoE performance with timing breakdown (decode mode, T=1)
#[test]
@@ -121,7 +51,7 @@ fn test_moe_e2e_decode_perf() {
.expect("router+topk fused kernel");
// Time fused Router+TopK
- let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
+ let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
router_topk.encode(
&x_buf,
@@ -179,7 +109,7 @@ fn test_moe_e2e_prefill_perf() {
.expect("router+topk fused kernel");
// Time fused Router+TopK
- let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {
+ let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
router_topk.encode(
&x_buf,
@@ -305,7 +235,7 @@ fn test_moe_pipeline_breakdown_decode() {
.expect("router+topk fused");
// Testing: Router + TopK + Counts+Offsets (FUSED)
- let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {
+ let router_topk_fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
router_topk_fused_kernel.encode(
&x_buf,
@@ -323,7 +253,7 @@ fn test_moe_pipeline_breakdown_decode() {
encoder.end_encoding().submit().wait_until_completed().unwrap();
});
- let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {
+ let counts_offsets_perf = run_perf_with_warmup("Counts+Offsets (FUSED)", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
counts_offsets_kernel.encode(
&topk_ids_buf,
@@ -338,7 +268,7 @@ fn test_moe_pipeline_breakdown_decode() {
encoder.end_encoding().submit().wait_until_completed().unwrap();
});
- let scatter_perf = time_kernel("Scatter", 2, 5, || {
+ let scatter_perf = run_perf_with_warmup("Scatter", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
scatter_bases_kernel.encode(
&partials_buf,
@@ -369,7 +299,7 @@ fn test_moe_pipeline_breakdown_decode() {
encoder.end_encoding().submit().wait_until_completed().unwrap();
});
- let gather_perf = time_kernel("Gather", 2, 5, || {
+ let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
gather_kernel.encode(
&mut encoder,
@@ -387,7 +317,7 @@ fn test_moe_pipeline_breakdown_decode() {
encoder.end_encoding().submit().wait_until_completed().unwrap();
});
- let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {
+ let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
experts_kernel.encode(
&mut encoder,
@@ -423,7 +353,7 @@ fn test_moe_pipeline_breakdown_decode() {
encoder.end_encoding().submit().wait_until_completed().unwrap();
});
- let finalize_perf = time_kernel("Finalize", 2, 5, || {
+ let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {
let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");
finalize_kernel.encode(
&tok2row_buf,