A high-performance inference engine for AI models
Models loading optimizations (#282)
Aleksandr Golokozf4a9f85c28c0
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/array.rsindex 13226241d..5229d6c34 100644--- a/crates/uzu/src/array.rs+++ b/crates/uzu/src/array.rs@@ -228,7 +228,7 @@ pub trait ArrayContextExt {label: &str,) -> Array<Self::Backend>;- fn create_array(+ fn create_array_zeros(&self,shape: &[usize],data_type: DataType,diff --git a/crates/uzu/src/audio/nanocodec/runtime.rs b/crates/uzu/src/audio/nanocodec/runtime.rsindex a7b340ce6..0c998470a 100644--- a/crates/uzu/src/audio/nanocodec/runtime.rs+++ b/crates/uzu/src/audio/nanocodec/runtime.rs@@ -324,17 +324,18 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {let kernel = <B::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32).map_err(|err| AudioError::Runtime(format!("failed to initialize fsq decode kernel: {err}")))?;- let mut tokens_array = context.create_array(+ let mut tokens_array = context.create_array_zeros(&[batch_size, self.config.num_groups(), frames],DataType::I32,"nanocodec_fsq_decode_tokens",);tokens_array.as_slice_mut::<i32>().copy_from_slice(&tokens_i32);- let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");+ let mut lengths_array =+ context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);- let output = context.create_array(+ let output = context.create_array_zeros(&[batch_size, self.config.channels(), frames],DataType::F32,"nanocodec_fsq_decode_output",@@ -414,17 +415,17 @@ impl<B: Backend> NanoCodecFsqRuntime<B> {let kernel = <B::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32).map_err(|err| AudioError::Runtime(format!("failed to initialize fsq encode kernel: {err}")))?;- let mut input = context.create_array(+ let mut input = context.create_array_zeros(&[batch_size, self.config.channels(), frames],DataType::F32,"nanocodec_fsq_encode_input",);input.as_slice_mut::<f32>().copy_from_slice(&padded_input);- let mut lengths = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);- let tokens = context.create_array(+ let tokens = context.create_array_zeros(&[batch_size, self.config.num_groups(), frames],DataType::I32,"nanocodec_fsq_encode_tokens",diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rsindex 034414d0c..3e2d90914 100644--- a/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs@@ -168,7 +168,7 @@ pub(super) fn read_norm_layer<B: Backend>(let bias = if use_bias {read_float_vector_exact::<B>(tree, "biases", channels, data_type)?} else {- context.create_array(&[channels], data_type, &format!("{label_prefix}_bias"))+ context.create_array_zeros(&[channels], data_type, &format!("{label_prefix}_bias"))};Ok(StructuredAudioNorm {scales,diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rsindex f07975c82..0c2ec5054 100644--- a/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs@@ -148,7 +148,7 @@ impl StructuredAudioCodecGraph {let mut lengths_array = resources.length_staging(batch_size);lengths_array.as_slice_mut::<i32>()[..lengths_i32.len()].copy_from_slice(&lengths_i32);- let output = context.create_array(+ let output = context.create_array_zeros(&[batch_size, frames, self.input_dim],quantizer_resources.data_type,"structured_audio_quantizer_output_nsc",diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rsindex 494ab745b..9e3891f0f 100644--- a/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs@@ -88,7 +88,7 @@ impl StructuredAudioCodecGraph {.collect::<AudioResult<Vec<_>>>()?;let context = resources.context().clone();return Ok(SubmittedDecodedPaddedAudio {- output: context.create_array(&[0], DataType::F32, "structured_audio_empty_decode_output"),+ output: context.create_array_zeros(&[0], DataType::F32, "structured_audio_empty_decode_output"),channels: 1,frames: out_lengths.iter().copied().max().unwrap_or(0),lengths: out_lengths,@@ -343,7 +343,7 @@ impl StructuredAudioCodecGraph {x = tanh_enqueue(&mut encoder,&x,- context.create_array(x.shape(), data_type, "structured_audio_tanh_output"),+ context.create_array_zeros(x.shape(), data_type, "structured_audio_tanh_output"),&kernels,)?;diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rsindex a30f99533..65c336e98 100644--- a/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs@@ -89,7 +89,7 @@ impl<B: Backend> DecodeWorkspace<B> {} else {"structured_audio_lengths_b"};- let array = context.create_array(&[min_elements], DataType::I32, label);+ let array = context.create_array_zeros(&[min_elements], DataType::I32, label);*slot = Some(array.clone());array}@@ -200,7 +200,7 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {return existing.clone();}}- let array = self.context.create_array(&[min_elements], DataType::U32, "quantizer_token_staging");+ let array = self.context.create_array_zeros(&[min_elements], DataType::U32, "quantizer_token_staging");*slot = Some(array.clone());array}@@ -215,7 +215,7 @@ impl<B: Backend> StructuredAudioRuntimeResources<B> {return existing.clone();}}- let array = self.context.create_array(&[min_elements], DataType::I32, "quantizer_length_staging");+ let array = self.context.create_array_zeros(&[min_elements], DataType::I32, "quantizer_length_staging");*slot = Some(array.clone());array}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex 054db8d82..b89f34fb7 100644--- a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -124,8 +124,8 @@ impl StructuredAudioCodecGraph {max_sequence_length,);let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(- context.clone(),- decoder_config,+ context.as_ref(),+ &decoder_config,&root_loader_view,transformer_subtree_name,);@@ -200,17 +200,17 @@ impl StructuredAudioCodecGraph {let residual_quantizers = self.config.n_codebooks;let residual_count_for_shape = residual_quantizers.max(1);let residual_codebook_rows_for_shape = self.codebook_size.max(1);- let residual_codebooks = context.create_array(+ let residual_codebooks = context.create_array_zeros(&[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim],data_type,"structured_audio_quantizer_residual_codebooks",);- let residual_out_proj = context.create_array(+ let residual_out_proj = context.create_array_zeros(&[residual_count_for_shape, self.input_dim, codebook_dim],data_type,"structured_audio_quantizer_residual_out_proj",);- let residual_out_bias = context.create_array(+ let residual_out_bias = context.create_array_zeros(&[residual_count_for_shape, self.input_dim],data_type,"structured_audio_quantizer_residual_out_bias",@@ -420,7 +420,7 @@ impl StructuredAudioCodecGraph {return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);}- let output = context.create_array(+ let output = context.create_array_zeros(&[batch_size, frames, self.input_dim],latent_nsc.data_type(),"structured_audio_post_module_output_nsc",diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 0feb27b2c..6b9f0bd02 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -8,6 +8,7 @@ use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{DataType,backends::common::{Backend, Encoder},+ config::ModelMetadata,encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,@@ -70,8 +71,11 @@ impl<B: Backend> ClassifierTrait for Classifier<B> {}impl<B: Backend> Classifier<B> {- pub fn new(model_path: &Path) -> Result<Self, Error> {- let context = ClassifierContext::new(model_path)?;+ pub fn new(+ model_path: &Path,+ model_metadata: &ModelMetadata,+ ) -> Result<Self, Error> {+ let context = ClassifierContext::new(model_path, model_metadata)?;Ok(Self {context,})diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rsindex dff68ae99..5ef7421b3 100644--- a/crates/uzu/src/classifier/classifier_context.rs+++ b/crates/uzu/src/classifier/classifier_context.rs@@ -1,4 +1,4 @@-use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};+use std::{cell::RefCell, fs::File, path::Path, rc::Rc};use crate::{DataType,@@ -36,17 +36,12 @@ pub struct ClassifierContext<B: Backend> {}impl<B: Backend> ClassifierContext<B> {- pub fn new(model_path: &Path) -> Result<Self, Error> {+ pub fn new(+ model_path: &Path,+ model_metadata: &ModelMetadata,+ ) -> Result<Self, Error> {let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;- let config_path = model_path.join("config.json");- if !config_path.exists() {- return Err(Error::UnableToLoadConfig);- }- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let model_metadata: ModelMetadata =- serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;-let classifier_model_config = model_metadata.model_config.as_classifier().ok_or(Error::UnableToLoadConfig)?;let decoder_config =diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex 8ee3c352b..b97125177 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -33,14 +33,14 @@ pub struct Decoder<B: Backend> {impl<B: Backend> Decoder<B> {pub fn new(- context: Rc<B::Context>,- decoder_config: Rc<DecoderConfig>,+ context: &B::Context,+ decoder_config: &DecoderConfig,root_weight_loader: &ParameterTree<B::Context>,) -> Self {let embedding_weight_loader = root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree");let embed = Embedding::new(- context.as_ref(),+ context,decoder_config.vocab_size as u32,decoder_config.model_dim as u32,&decoder_config.embedding_config,@@ -61,8 +61,8 @@ impl<B: Backend> Decoder<B> {/// Used by models whose token lookup weights and logits readout weights#[cfg(all(feature = "audio-runtime", metal_backend))]pub fn new_with_embedding_and_readout_subtrees(- context: Rc<B::Context>,- decoder_config: Rc<DecoderConfig>,+ context: &B::Context,+ decoder_config: &DecoderConfig,root_weight_loader: &ParameterTree<B::Context>,transformer_subtree: &str,embedding_subtree: &str,@@ -73,7 +73,7 @@ impl<B: Backend> Decoder<B> {let readout_weight_loader = root_weight_loader.subtree(readout_subtree).expect("Failed to get readout subtree");let embed = Embedding::new_with_lookup_and_readout_trees(- context.as_ref(),+ context,decoder_config.vocab_size as u32,decoder_config.model_dim as u32,&decoder_config.embedding_config,@@ -82,12 +82,8 @@ impl<B: Backend> Decoder<B> {).expect("Failed to create embedding");- let (layers, norm) = Self::build_transformer_layers_and_norm(- context,- decoder_config.clone(),- root_weight_loader,- transformer_subtree,- );+ let (layers, norm) =+ Self::build_transformer_layers_and_norm(context, decoder_config, root_weight_loader, transformer_subtree);Self {embed,@@ -97,8 +93,8 @@ impl<B: Backend> Decoder<B> {}pub(crate) fn build_transformer_layers_and_norm(- context: Rc<B::Context>,- decoder_config: Rc<DecoderConfig>,+ context: &B::Context,+ decoder_config: &DecoderConfig,root_weight_loader: &ParameterTree<B::Context>,transformer_subtree: &str,) -> (Box<[LayerExecutables<B>]>, RMSNorm<B>) {@@ -145,13 +141,13 @@ impl<B: Backend> Decoder<B> {let layer_type = model_shape.layer_type(layer_index);let rope_for_layer = match layer_type {DecoderLayerType::Transformer => {- let mut rope_block = global_rope.clone().expect("Global rope missing for transformer layer");- if let (Some(_), Some(local_rope_block)) =- (sliding_window_sizes[layer_index], local_rope.clone())+ if let Some(_) = sliding_window_sizes[layer_index]+ && let Some(local_rope_block) = local_rope.clone(){- rope_block = local_rope_block;+ Some(local_rope_block)+ } else {+ Some(global_rope.clone().expect("Global rope missing for transformer layer"))}- Some(rope_block)},DecoderLayerType::StateSpace {..@@ -164,7 +160,7 @@ impl<B: Backend> Decoder<B> {let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();LayerExecutables::new(- context.clone(),+ context,layer_config,layer_type,layer_index,@@ -181,7 +177,7 @@ impl<B: Backend> Decoder<B> {.collect::<Vec<_>>();let norm_block = RMSNorm::new(- context.as_ref(),+ context,norm_data_type,decoder_config.output_norm_config.clone(),ArrayId::Main,diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex 7c5ce2540..cde4dda7a 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -32,7 +32,7 @@ pub struct LayerExecutables<B: Backend> {impl<B: Backend> LayerExecutables<B> {#[allow(clippy::too_many_arguments)]pub fn new(- context: Rc<B::Context>,+ context: &B::Context,layer_config: &DecoderLayerConfig,layer_type: &DecoderLayerType,layer_index: usize,@@ -45,21 +45,20 @@ impl<B: Backend> LayerExecutables<B> {decoder_layer_loader: &ParameterTree<B::Context>,rope: Option<Rc<Rope<B>>>,) -> Self {- let ctx = context.as_ref(); // Reference for functions expecting &B::Contextlet intermediate_data_type: DataType = match &layer_config.mixer_config {MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),};let copy_main_to_shortcut = TensorCopy::<B>::new(- ctx,+ context,intermediate_data_type,vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),).unwrap();let pre_attention_norm = RMSNorm::new(- ctx,+ context,intermediate_data_type,layer_config.pre_attention_norm_config.clone(),ArrayId::Main,@@ -77,7 +76,7 @@ impl<B: Backend> LayerExecutables<B> {attention_config.has_qkv_biases,model_dim,[num_heads * head_dim, num_groups * head_dim, num_groups * head_dim],- ctx,+ context,&decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),ArrayId::Main,ArrayId::QKV,@@ -91,7 +90,7 @@ impl<B: Backend> LayerExecutables<B> {false,model_dim,[num_heads * head_dim],- ctx,+ context,&decoder_layer_loader.subtree("mixer.gate_projection").unwrap(),ArrayId::Main,ArrayId::Gate,@@ -105,7 +104,7 @@ impl<B: Backend> LayerExecutables<B> {let qk_norm =if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {match QKNorm::new(- ctx,+ context,intermediate_data_type,attention_config.query_norm_config.clone(),attention_config.key_norm_config.clone(),@@ -127,7 +126,7 @@ impl<B: Backend> LayerExecutables<B> {attention_config.has_out_biases,num_heads * head_dim,[model_dim],- ctx,+ context,&decoder_layer_loader.subtree("mixer.out_projection").unwrap(),ArrayId::AttentionOutput,ArrayId::Main,@@ -135,7 +134,7 @@ impl<B: Backend> LayerExecutables<B> {.expect("Failed to create out projection");let attention = Attention::new(- ctx,+ context,intermediate_data_type,layer_index,attention_scale,@@ -157,7 +156,7 @@ impl<B: Backend> LayerExecutables<B> {},MixerConfig::Mamba(mamba_config) => {let mixer = MambaMixer::new(- ctx,+ context,layer_type.clone(),mamba_config.clone(),layer_index,@@ -173,7 +172,7 @@ impl<B: Backend> LayerExecutables<B> {},MixerConfig::ShortConv(short_conv_config) => {let mixer = ShortConvMixer::new(- ctx,+ context,layer_type.clone(),short_conv_config.clone(),layer_index,@@ -189,7 +188,7 @@ impl<B: Backend> LayerExecutables<B> {let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {Some(RMSNorm::new(- ctx,+ context,intermediate_data_type,norm_config.clone(),ArrayId::Main,@@ -203,14 +202,14 @@ impl<B: Backend> LayerExecutables<B> {};let main_shortcut_add_swap = TensorAddSwap::<B>::new(- ctx,+ context,intermediate_data_type,vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),).unwrap();let pre_mlp_norm = RMSNorm::new(- ctx,+ context,intermediate_data_type,layer_config.pre_mlp_norm_config.clone(),ArrayId::Main,@@ -223,7 +222,7 @@ impl<B: Backend> LayerExecutables<B> {&layer_config.mlp_config,model_dim,hidden_dim,- context.as_ref(),+ context,&decoder_layer_loader.subtree("mlp").unwrap(),).expect("Failed to create mlp block");@@ -231,7 +230,7 @@ impl<B: Backend> LayerExecutables<B> {let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(RMSNorm::new(- ctx,+ context,intermediate_data_type,norm_config.clone(),ArrayId::Main,diff --git a/crates/uzu/src/forward_pass/cache_layers.rs b/crates/uzu/src/forward_pass/cache_layers.rsindex 5fdbcdbd4..b000fe01b 100644--- a/crates/uzu/src/forward_pass/cache_layers.rs+++ b/crates/uzu/src/forward_pass/cache_layers.rs@@ -123,12 +123,12 @@ impl<B: Backend> CacheLayers<B> {CacheLayer::Transformer(KVCacheLayer {state: state.clone(),- keys: context.create_array(+ keys: context.create_array_zeros(&shape,model_shape.kv_cache_data_type(),&format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),),- values: context.create_array(+ values: context.create_array_zeros(&shape,model_shape.kv_cache_data_type(),&format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),@@ -158,12 +158,12 @@ impl<B: Backend> CacheLayers<B> {let dtype = model_shape.activation_data_type();CacheLayer::StateSpace(SSMLayer {- conv_state: context.create_array(+ conv_state: context.create_array_zeros(&conv_shape,dtype,&format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),),- ssm_state: context.create_array(+ ssm_state: context.create_array_zeros(&ssm_shape,dtype,&format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),@@ -179,12 +179,12 @@ impl<B: Backend> CacheLayers<B> {let dtype = model_shape.activation_data_type();CacheLayer::ShortConv(ShortConvLayer {- conv_state: context.create_array(+ conv_state: context.create_array_zeros(&conv_shape,dtype,&format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),),- suffix_state: context.create_array(+ suffix_state: context.create_array_zeros(&suffix_state_shape,dtype,&format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),@@ -374,12 +374,12 @@ impl<B: Backend> CacheLayers<B> {}let new_shape = [num_groups, new_total_len, head_dim];- let mut new_keys = context.create_array(+ let mut new_keys = context.create_array_zeros(&new_shape,dtype,&format!("{ARRAY_TRANSFORMER_KEYS_LABEL}_{layer_index}"),);- let mut new_values = context.create_array(+ let mut new_values = context.create_array_zeros(&new_shape,dtype,&format!("{ARRAY_TRANSFORMER_VALUES_LABEL}_{layer_index}"),@@ -401,7 +401,7 @@ impl<B: Backend> CacheLayers<B> {CacheLayer::StateSpace(layer) => {let conv_shape = layer.conv_state.shape().to_vec();let conv_dtype = layer.conv_state.data_type();- let mut new_conv = context.create_array(+ let mut new_conv = context.create_array_uninitialized(&conv_shape,conv_dtype,&format!("{ARRAY_STATE_SPACE_CONV_STATE_LABEL}_{layer_index}"),@@ -410,7 +410,7 @@ impl<B: Backend> CacheLayers<B> {let ssm_shape = layer.ssm_state.shape().to_vec();let ssm_dtype = layer.ssm_state.data_type();- let mut new_ssm = context.create_array(+ let mut new_ssm = context.create_array_uninitialized(&ssm_shape,ssm_dtype,&format!("{ARRAY_STATE_SPACE_SSM_STATE_LABEL}_{layer_index}"),@@ -425,7 +425,7 @@ impl<B: Backend> CacheLayers<B> {CacheLayer::ShortConv(layer) => {let conv_shape = layer.conv_state.shape().to_vec();let conv_dtype = layer.conv_state.data_type();- let mut new_conv = context.create_array(+ let mut new_conv = context.create_array_uninitialized(&conv_shape,conv_dtype,&format!("{ARRAY_SHORT_CONV_CONV_STATE_LABEL}_{layer_index}"),@@ -434,14 +434,11 @@ impl<B: Backend> CacheLayers<B> {let suffix_shape = layer.suffix_state.shape().to_vec();let suffix_dtype = layer.suffix_state.data_type();- let mut new_suffix = context.create_array(+ let new_suffix = context.create_array_zeros(&suffix_shape,suffix_dtype,&format!("{ARRAY_SHORT_CONV_SUFFIX_STATE_LABEL}_{layer_index}"),);- {- new_suffix.as_bytes_mut().fill(0);- }CacheLayer::ShortConv(ShortConvLayer {conv_state: new_conv,diff --git a/crates/uzu/src/forward_pass/kv_cache_layer.rs b/crates/uzu/src/forward_pass/kv_cache_layer.rsindex 086c256e8..b31ca92c3 100644--- a/crates/uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/uzu/src/forward_pass/kv_cache_layer.rs@@ -325,8 +325,10 @@ impl<B: Backend> KVCacheLayer<B> {let dtype = self.keys.data_type();let slice_shape = [num_groups, len, head_dim];- let mut slice_keys = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_keys");- let mut slice_values = context.create_array(&slice_shape, dtype, "kv_cache_layer_slice_values");+ let mut slice_keys =+ context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_keys");+ let mut slice_values =+ context.create_array_uninitialized(&slice_shape, dtype, "kv_cache_layer_slice_values");let slots: Vec<usize> = (range.start..range.end).enumerate()diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 7e93aedf6..5ad622507 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -20,6 +20,7 @@ use crate::{Backend, Buffer, CommandBuffer, Context, Encoder, Executable,kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},},+ config::ModelMetadata,encodable_block::EncodingParameters,forward_pass::{cache_layers::{CacheLayer, CacheLayersSlice},@@ -689,10 +690,11 @@ impl<B: Backend> LanguageModelGenerator<B> {pub fn new(model_path: &Path,decoding_config: DecodingConfig,+ model_metadata: &ModelMetadata,) -> Result<Self, Error> {let gpu_capture = GpuCaptureManager::new();- let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;+ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config, model_metadata)?;Ok(Self {decoding_config,diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex b210b660d..1849f66d3 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -1,7 +1,6 @@use std::{cell::{Cell, RefCell},fs::File,- io::BufReader,path::Path,rc::Rc,};@@ -119,7 +118,7 @@ pub struct LanguageModelGeneratorContext<B: Backend> {pub scratch_buffers: ScratchBuffers<B>,pub model_config: LanguageModelConfig,- pub decoder_config: Rc<DecoderConfig>,+ pub decoder_config: DecoderConfig,pub model_shape: ModelShape,pub executables: Decoder<B>,pub kv_cache_update: Box<KVCacheUpdate<B>>,@@ -139,21 +138,14 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {pub fn new(model_path: &Path,decoding_config: &DecodingConfig,+ model_metadata: &ModelMetadata,) -> Result<Self, Error> {let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;- let config_path = model_path.join("config.json");- if !config_path.exists() {- return Err(Error::UnableToLoadConfig);- }- let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;- let model_metadata: ModelMetadata =- serde_json::from_reader(BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;-// Extract language model configlet language_model_config = model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;- let decoder_config = Rc::new(language_model_config.decoder_config().map_err(|_| Error::UnableToLoadConfig)?);+ let decoder_config = language_model_config.decoder_config().map_err(|_| Error::UnableToLoadConfig)?;let model_shape = ModelShape::from_decoder_config(&decoder_config);let prefill_step_size = decoding_config.prefill_step_size.resolve(language_model_config);@@ -175,7 +167,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let scratch_buffers =ScratchBuffers::new(context.as_ref(), &decoder_config, &model_shape, max_prefix_length, max_suffix_length);- let executables = Decoder::new(context.clone(), decoder_config.clone(), &root_loader_view);+ let executables = Decoder::new(context.as_ref(), &decoder_config, &root_loader_view);let cache_layers = Rc::new(RefCell::new(CacheLayers::new(context.as_ref(),diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rsindex 529a8361e..cbe485766 100644--- a/crates/uzu/src/parameters/loader.rs+++ b/crates/uzu/src/parameters/loader.rs@@ -113,16 +113,17 @@ wherelet (offset, size) = (metadata_entry.offset, metadata_entry.size);let array_key = key.replace(".", "_");let array_label = format!("parameter_loader_{array_key}");- let mut array = self.context.create_array(&metadata_entry.shape, metadata_entry.data_type, &array_label);- let expected_size = array.size();- if expected_size != size {+ let mut array =+ self.context.create_array_uninitialized(&metadata_entry.shape, metadata_entry.data_type, &array_label);+ if array.size() != size {return Err(ParameterLoaderError::SizeMismatch {data_type: metadata_entry.data_type,shape: metadata_entry.shape.to_owned(),- expected_size,+ expected_size: array.size(),actual_size: size,});}+self.file.read_exact_at(array.as_bytes_mut(), offset as u64)?;Ok(array)}diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex d8031c18a..5854e5e2b 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -137,8 +137,10 @@ impl ChatSession {let output_parser =OutputParser::new(language_model_config.message_processor_config.output_parser_regex.clone())?;- let llm =- Box::new(LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone()).map_err(Error::from)?);+ let llm = Box::new(+ LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone(), &model_metadata)+ .map_err(Error::from)?,+ );Ok(Self {model_path,diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rsindex cc3de24aa..f863ab7f4 100644--- a/crates/uzu/src/session/classification_session.rs+++ b/crates/uzu/src/session/classification_session.rs@@ -47,7 +47,7 @@ impl ClassificationSession {let input_processor =Box::new(InputProcessorDefault::new(classifier_model_config.message_processor_config.clone()))as Box<dyn InputProcessor>;- let classifier = Box::new(Classifier::<B>::new(&model_path)?);+ let classifier = Box::new(Classifier::<B>::new(&model_path, &model_metadata)?);Ok(Self {model_path,diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rsindex a236b2207..3c0b43a59 100644--- a/crates/uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs@@ -37,8 +37,8 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {let scratch_buffers =ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_prefix_length, max_suffix_length);let executables = Decoder::new_with_embedding_and_readout_subtrees(- context.clone(),- decoder_config.clone(),+ context.as_ref(),+ &decoder_config,&root_loader_view,transformer_subtree,embedding_subtree,@@ -236,9 +236,9 @@ impl<B: Backend> TokenDecoderRunner<B> {<B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type).map_err(unable_to_create_context)?;let single_hidden_capture =- context.create_array(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");+ context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");let single_override_embedding =- context.create_array(&[1, model_dim], activation_data_type, "tts_single_override_embedding");+ context.create_array_zeros(&[1, model_dim], activation_data_type, "tts_single_override_embedding");Ok(Self {ctx,diff --git a/crates/uzu/src/session/tts_session/fishaudio.rs b/crates/uzu/src/session/tts_session/fishaudio.rsindex 52afd9d61..9ad5efdb2 100644--- a/crates/uzu/src/session/tts_session/fishaudio.rs+++ b/crates/uzu/src/session/tts_session/fishaudio.rs@@ -109,7 +109,7 @@ impl<B: Backend> FishAudioSemanticBridge<B> {let tensor_copy =<B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;let codebook_token_indices =- context.create_array(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");+ context.create_array_zeros(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");Ok(Self {embedding_rows_sum,diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex b134c9fa7..248b9cac8 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -185,7 +185,7 @@ impl<B: Backend> TraceValidator<B> {serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;let context = if metadata.model_config.is_classifier() {- let classifier = Classifier::new(model_path)?;+ let classifier = Classifier::new(model_path, &metadata)?;ModelContext::Classifier(classifier)} else {let prefill_step_size = Self::determine_prefill_step_size(model_path);@@ -198,7 +198,7 @@ impl<B: Backend> TraceValidator<B> {AsyncBatchSize::default(),false,);- let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;+ let mut llm_context = LanguageModelGeneratorContext::new(model_path, &decoding_config, &metadata)?;let desired_suffix_length = prefill_step_size.max(decoding_config.generate_suffix_length());Self::ensure_llm_context_capacity(&decoding_config, desired_suffix_length, &mut llm_context);ModelContext::LanguageModelGenerator(llm_context)diff --git a/crates/uzu/tests/audio_kernel_perf_test.rs b/crates/uzu/tests/audio_kernel_perf_test.rsindex 034bf919c..5a2653ef8 100644--- a/crates/uzu/tests/audio_kernel_perf_test.rs+++ b/crates/uzu/tests/audio_kernel_perf_test.rs@@ -121,17 +121,21 @@ fn audio_kernel_perf() {let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();- let tokens = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");+ let tokens =+ context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");let lengths = make_lengths(&context, batch_size, frames as i32);- let semantic_codebook = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");- let semantic_out_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");- let semantic_out_bias = context.create_array(&[INPUT_DIM], dt, "perf_sem_bias");- let residual_codebooks =- context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "perf_res_cbs");+ let semantic_codebook = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");+ let semantic_out_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");+ let semantic_out_bias = context.create_array_zeros(&[INPUT_DIM], dt, "perf_sem_bias");+ let residual_codebooks = context.create_array_zeros(+ &[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM],+ dt,+ "perf_res_cbs",+ );let residual_out_proj =- context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");- let residual_out_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");- let output = context.create_array(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");+ context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");+ let residual_out_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");+ let output = context.create_array_zeros(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");let label = format!("QuantizerDecode [B={}, T={}, K={}, dim={}, cdim={}]",@@ -204,10 +208,10 @@ fn audio_kernel_perf() {let seq_in = frames;let seq_out = seq_in * stride;let ksize = stride * 2;- let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");- let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");- let bias = context.create_array(&[cout], dt, "tconv_b");- let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");+ let bias = context.create_array_zeros(&[cout], dt, "tconv_b");+ let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");let lengths = make_lengths(&context, batch_size, seq_out as i32);let label = format!("TransConv {tag} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");@@ -257,10 +261,10 @@ fn audio_kernel_perf() {let seq_in = frames;let seq_out = seq_in * stride;let ksize = stride * 2;- let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");- let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");- let bias = context.create_array(&[cout], dt, "tconv_b");- let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");+ let input = context.create_array_zeros(&[batch_size * cin * seq_in], dt, "tconv_in");+ let weight = context.create_array_zeros(&[cin * cout * ksize], dt, "tconv_w");+ let bias = context.create_array_zeros(&[cout], dt, "tconv_b");+ let output = context.create_array_zeros(&[batch_size * cout * seq_out], dt, "tconv_out");let lengths = make_lengths(&context, batch_size, seq_out as i32);let label = format!("TransConv dec{block_idx} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");@@ -317,10 +321,10 @@ fn audio_kernel_perf() {let cin = UP_CHANNELS[2]; // 256let cout = DECODER_DIM; // 1536let ksize = 7;- let input = context.create_array(&[batch_size * cin * fc_seq], dt, "fc_in");- let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");- let bias = context.create_array(&[cout], dt, "fc_b");- let output = context.create_array(&[batch_size * cout * fc_seq], dt, "fc_out");+ let input = context.create_array_zeros(&[batch_size * cin * fc_seq], dt, "fc_in");+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");+ let bias = context.create_array_zeros(&[cout], dt, "fc_b");+ let output = context.create_array_zeros(&[batch_size * cout * fc_seq], dt, "fc_out");let lengths = make_lengths(&context, batch_size, fc_seq as i32);let label = format!("CausalConv1d first_conv [{cin}->{cout}, k={ksize}, T={fc_seq}]");@@ -368,10 +372,10 @@ fn audio_kernel_perf() {for &dilation in &[1, 3, 9] {let label =format!("CausalConv1d res dec{block_idx} [{ch}->{ch}, k={ksize}, d={dilation}, T={dec_frames}]");- let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");- let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");- let bias = context.create_array(&[ch], dt, "res_b");- let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");+ let bias = context.create_array_zeros(&[ch], dt, "res_b");+ let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");let lengths = make_lengths(&context, batch_size, dec_frames as i32);let ms = measure(&label, || {@@ -415,10 +419,10 @@ fn audio_kernel_perf() {let cout = 1;let ksize = 7;let final_frames = SEMANTIC_FRAMES_10S * 2 * 2 * 8 * 8 * 4 * 2; // full resolution- let input = context.create_array(&[batch_size * cin * final_frames], dt, "fc_in");- let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");- let bias = context.create_array(&[cout], dt, "fc_b");- let output = context.create_array(&[batch_size * cout * final_frames], dt, "fc_out");+ let input = context.create_array_zeros(&[batch_size * cin * final_frames], dt, "fc_in");+ let weight = context.create_array_zeros(&[cout * cin * ksize], dt, "fc_w");+ let bias = context.create_array_zeros(&[cout], dt, "fc_b");+ let output = context.create_array_zeros(&[batch_size * cout * final_frames], dt, "fc_out");let lengths = make_lengths(&context, batch_size, final_frames as i32);let label = format!("CausalConv1d final_conv [{cin}->{cout}, k={ksize}, T={final_frames}]");@@ -473,11 +477,11 @@ fn audio_kernel_perf() {let ksize = 7;let label = format!("CausalConv1dResidual dec{block_idx} [{ch}->{ch}, k={ksize}, T={dec_frames}]");- let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");- let residual = context.create_array(&[batch_size * ch * dec_frames], dt, "res_res");- let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");- let bias = context.create_array(&[ch], dt, "res_b");- let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");+ let input = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_in");+ let residual = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_res");+ let weight = context.create_array_zeros(&[ch * ch * ksize], dt, "res_w");+ let bias = context.create_array_zeros(&[ch], dt, "res_b");+ let output = context.create_array_zeros(&[batch_size * ch * dec_frames], dt, "res_out");let lengths = make_lengths(&context, batch_size, dec_frames as i32);let ms = measure(&label, || {@@ -532,10 +536,10 @@ fn audio_kernel_perf() {let ksize = 7;let label = format!("CausalConv1dGrouped dw up{block_idx} [{ch}, g={ch}, k={ksize}, T={frames}]");- let input = context.create_array(&[batch_size * ch * frames], dt, "dw_in");- let weight = context.create_array(&[ch * 1 * ksize], dt, "dw_w");- let bias = context.create_array(&[ch], dt, "dw_b");- let output = context.create_array(&[batch_size * ch * frames], dt, "dw_out");+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_in");+ let weight = context.create_array_zeros(&[ch * 1 * ksize], dt, "dw_w");+ let bias = context.create_array_zeros(&[ch], dt, "dw_b");+ let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "dw_out");let lengths = make_lengths(&context, batch_size, frames as i32);let ms = measure(&label, || {@@ -588,9 +592,9 @@ fn audio_kernel_perf() {let ch_in = DEC_CHANNELS[block_idx];{let label = format!("HalfSnake pre-tconv dec{block_idx} [{ch_in}ch, T={dec_frames}]");- let input = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_in");- let alpha = context.create_array(&[ch_in], dt, "sn_a");- let output = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_out");+ let input = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_in");+ let alpha = context.create_array_zeros(&[ch_in], dt, "sn_a");+ let output = context.create_array_zeros(&[batch_size * ch_in * dec_frames], dt, "sn_out");let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();@@ -624,9 +628,9 @@ fn audio_kernel_perf() {let ch_out = DEC_CHANNELS[block_idx + 1];{let label = format!("HalfSnake res dec{block_idx} [{ch_out}ch, T={dec_frames}]");- let input = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_in");- let alpha = context.create_array(&[ch_out], dt, "sn_a");- let output = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_out");+ let input = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_in");+ let alpha = context.create_array_zeros(&[ch_out], dt, "sn_a");+ let output = context.create_array_zeros(&[batch_size * ch_out * dec_frames], dt, "sn_out");let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();@@ -673,10 +677,10 @@ fn audio_kernel_perf() {// Norm{let label = format!("NormNcs up{block_idx} [{ch}ch, T={frames}]");- let input = context.create_array(&[batch_size * ch * frames], dt, "norm_in");- let scales = context.create_array(&[ch], dt, "norm_s");- let bias = context.create_array(&[ch], dt, "norm_b");- let output = context.create_array(&[batch_size * ch * frames], dt, "norm_out");+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_in");+ let scales = context.create_array_zeros(&[ch], dt, "norm_s");+ let bias = context.create_array_zeros(&[ch], dt, "norm_b");+ let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "norm_out");let lengths = make_lengths(&context, batch_size, frames as i32);let ms = measure(&label, || {@@ -714,10 +718,10 @@ fn audio_kernel_perf() {// Pointwise conv1 (ch -> ch){let label = format!("Conv1d pw up{block_idx} [{ch}->{ch}, k=1, T={frames}]");- let input = context.create_array(&[batch_size * ch * frames], dt, "pw_in");- let weight = context.create_array(&[ch * ch], dt, "pw_w");- let bias = context.create_array(&[ch], dt, "pw_b");- let output = context.create_array(&[batch_size * ch * frames], dt, "pw_out");+ let input = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_in");+ let weight = context.create_array_zeros(&[ch * ch], dt, "pw_w");+ let bias = context.create_array_zeros(&[ch], dt, "pw_b");+ let output = context.create_array_zeros(&[batch_size * ch * frames], dt, "pw_out");let lengths = make_lengths(&context, batch_size, frames as i32);let ms = measure(&label, || {@@ -761,8 +765,8 @@ fn audio_kernel_perf() {{let n = ch * frames;let label = format!("GELU up{block_idx} [n={n}]");- let input = context.create_array(&[n], dt, "gelu_in");- let output = context.create_array(&[n], dt, "gelu_out");+ let input = context.create_array_zeros(&[n], dt, "gelu_in");+ let output = context.create_array_zeros(&[n], dt, "gelu_out");let ms = measure(&label, || {let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();@@ -809,15 +813,15 @@ fn audio_kernel_perf() {let frames = SEMANTIC_FRAMES_10S;// Quantizer arrays- let p_q_tok = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");+ let p_q_tok = context.create_array_zeros(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");let p_q_len = make_lengths(&context, batch_size, frames as i32);- let p_q_sem_cb = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");- let p_q_sem_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");- let p_q_sem_bias = context.create_array(&[INPUT_DIM], dt, "p_sem_bias");+ let p_q_sem_cb = context.create_array_zeros(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");+ let p_q_sem_proj = context.create_array_zeros(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");+ let p_q_sem_bias = context.create_array_zeros(&[INPUT_DIM], dt, "p_sem_bias");let p_q_res_cbs =- context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");- let p_q_res_proj = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");- let p_q_res_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");+ context.create_array_zeros(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");+ let p_q_res_proj = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");+ let p_q_res_bias = context.create_array_zeros(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");// Compute frame counts at each stagelet f_up0 = frames * UP_STRIDES[0]; // 430@@ -830,42 +834,42 @@ fn audio_kernel_perf() {// Pipeline scratch: large enough for the biggest intermediate// Biggest is decoder_dim * f_up1 or ch * f_dec stageslet max_elems: usize = batch_size * DECODER_DIM * f_dec3.max(f_up1);- let scratch_a = context.create_array(&[max_elems], dt, "pipe_sa");- let scratch_b = context.create_array(&[max_elems], dt, "pipe_sb");+ let scratch_a = context.create_array_zeros(&[max_elems], dt, "pipe_sa");+ let scratch_b = context.create_array_zeros(&[max_elems], dt, "pipe_sb");// Third scratch buffer for residual connections (avoids RefCell borrow conflict)- let scratch_r = context.create_array(&[max_elems], dt, "pipe_sr");+ let scratch_r = context.create_array_zeros(&[max_elems], dt, "pipe_sr");// Upsample block 0: 1024->512, stride=2- let u0_w = context.create_array(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");- let u0_b = context.create_array(&[UP_CHANNELS[1]], dt, "u0_b");+ let u0_w = context.create_array_zeros(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");+ let u0_b = context.create_array_zeros(&[UP_CHANNELS[1]], dt, "u0_b");// ConvNeXt 0 (ch=512)let c0_ch = UP_CHANNELS[1];- let c0_dw_w = context.create_array(&[c0_ch * 1 * 7], dt, "c0_dw_w");- let c0_dw_b = context.create_array(&[c0_ch], dt, "c0_dw_b");- let c0_ns = context.create_array(&[c0_ch], dt, "c0_ns");- let c0_nb = context.create_array(&[c0_ch], dt, "c0_nb");- let c0_p1_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p1_w");- let c0_p1_b = context.create_array(&[c0_ch], dt, "c0_p1_b");- let c0_p2_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p2_w");- let c0_p2_b = context.create_array(&[c0_ch], dt, "c0_p2_b");+ let c0_dw_w = context.create_array_zeros(&[c0_ch * 1 * 7], dt, "c0_dw_w");+ let c0_dw_b = context.create_array_zeros(&[c0_ch], dt, "c0_dw_b");+ let c0_ns = context.create_array_zeros(&[c0_ch], dt, "c0_ns");+ let c0_nb = context.create_array_zeros(&[c0_ch], dt, "c0_nb");+ let c0_p1_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p1_w");+ let c0_p1_b = context.create_array_zeros(&[c0_ch], dt, "c0_p1_b");+ let c0_p2_w = context.create_array_zeros(&[c0_ch * c0_ch], dt, "c0_p2_w");+ let c0_p2_b = context.create_array_zeros(&[c0_ch], dt, "c0_p2_b");// Upsample block 1: 512->256, stride=2- let u1_w = context.create_array(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");- let u1_b = context.create_array(&[UP_CHANNELS[2]], dt, "u1_b");+ let u1_w = context.create_array_zeros(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");+ let u1_b = context.create_array_zeros(&[UP_CHANNELS[2]], dt, "u1_b");// ConvNeXt 1 (ch=256)let c1_ch = UP_CHANNELS[2];- let c1_dw_w = context.create_array(&[c1_ch * 1 * 7], dt, "c1_dw_w");- let c1_dw_b = context.create_array(&[c1_ch], dt, "c1_dw_b");- let c1_ns = context.create_array(&[c1_ch], dt, "c1_ns");- let c1_nb = context.create_array(&[c1_ch], dt, "c1_nb");- let c1_p1_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p1_w");- let c1_p1_b = context.create_array(&[c1_ch], dt, "c1_p1_b");- let c1_p2_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p2_w");- let c1_p2_b = context.create_array(&[c1_ch], dt, "c1_p2_b");+ let c1_dw_w = context.create_array_zeros(&[c1_ch * 1 * 7], dt, "c1_dw_w");+ let c1_dw_b = context.create_array_zeros(&[c1_ch], dt, "c1_dw_b");+ let c1_ns = context.create_array_zeros(&[c1_ch], dt, "c1_ns");+ let c1_nb = context.create_array_zeros(&[c1_ch], dt, "c1_nb");+ let c1_p1_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p1_w");+ let c1_p1_b = context.create_array_zeros(&[c1_ch], dt, "c1_p1_b");+ let c1_p2_w = context.create_array_zeros(&[c1_ch * c1_ch], dt, "c1_p2_w");+ let c1_p2_b = context.create_array_zeros(&[c1_ch], dt, "c1_p2_b");// first_conv: 256->1536, k=7- let fc_w = context.create_array(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");- let fc_b = context.create_array(&[DECODER_DIM], dt, "fc_b");+ let fc_w = context.create_array_zeros(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");+ let fc_b = context.create_array_zeros(&[DECODER_DIM], dt, "fc_b");// Decoder block weights for all 4 blocksstruct DecBlockWeights {@@ -888,27 +892,27 @@ fn audio_kernel_perf() {let stride = DEC_STRIDES[i];let ksize = stride * 2;DecBlockWeights {- snake_a: context.create_array(&[cin], dt, &format!("d{i}_sa")),- tconv_w: context.create_array(&[cin * cout * ksize], dt, &format!("d{i}_tw")),- tconv_b: context.create_array(&[cout], dt, &format!("d{i}_tb")),- ru_s1: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s1"))),+ snake_a: context.create_array_zeros(&[cin], dt, &format!("d{i}_sa")),+ tconv_w: context.create_array_zeros(&[cin * cout * ksize], dt, &format!("d{i}_tw")),+ tconv_b: context.create_array_zeros(&[cout], dt, &format!("d{i}_tb")),+ ru_s1: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s1"))),ru_c1_w: std::array::from_fn(|j| {- context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))+ context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))}),- ru_c1_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c1b"))),- ru_s2: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s2"))),+ ru_c1_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c1b"))),+ ru_s2: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_s2"))),ru_c2_w: std::array::from_fn(|j| {- context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))+ context.create_array_zeros(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))}),- ru_c2_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c2b"))),+ ru_c2_b: std::array::from_fn(|j| context.create_array_zeros(&[cout], dt, &format!("d{i}_r{j}_c2b"))),}}).collect();// final snake + conv + tanh- let final_snake_a = context.create_array(&[DEC_CHANNELS[4]], dt, "final_sa");- let final_conv_w = context.create_array(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");- let final_conv_b = context.create_array(&[1], dt, "final_cb");+ let final_snake_a = context.create_array_zeros(&[DEC_CHANNELS[4]], dt, "final_sa");+ let final_conv_w = context.create_array_zeros(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");+ let final_conv_b = context.create_array_zeros(&[1], dt, "final_cb");// Length arrayslet len_up0 = make_lengths(&context, batch_size, f_up0 as i32);diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex ac66869ed..42407e933 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -3,6 +3,8 @@pub mod assert;pub mod audio_nanocodec_fsq_reference;pub mod audio_nanocodec_ops_reference;+pub mod path;+pub mod perf;pub mod proptest;/// Invokes `$body` once per available backend, with `$B` bound to each backend type.diff --git a/crates/uzu/tests/integration/util/path.rs b/crates/uzu/tests/common/path.rssimilarity index 100%rename from crates/uzu/tests/integration/util/path.rsrename to crates/uzu/tests/common/path.rsdiff --git a/crates/uzu/tests/common/perf.rs b/crates/uzu/tests/common/perf.rsnew file mode 100644index 000000000..bf4db4c8c--- /dev/null+++ b/crates/uzu/tests/common/perf.rs@@ -0,0 +1,75 @@+use std::time::Instant;++pub struct PerfResult {+ pub name: String,+ pub mean_ms: f64,+ pub median_ms: f64,+ pub min_ms: f64,+ pub max_ms: f64,+ pub std_dev_ms: f64,+}++impl PerfResult {+ fn new(+ name: String,+ times_ms: &[f64],+ ) -> Self {+ let mut sorted = times_ms.to_vec();+ sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());++ let mean = sorted.iter().sum::<f64>() / sorted.len() as f64;+ let median = sorted[sorted.len() / 2];+ let min = sorted[0];+ let max = sorted[sorted.len() - 1];++ let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / sorted.len() as f64;+ let std_dev = variance.sqrt();++ Self {+ name,+ mean_ms: mean,+ median_ms: median,+ min_ms: min,+ max_ms: max,+ std_dev_ms: std_dev,+ }+ }++ pub fn print(&self) {+ eprintln!(+ " {:<30} mean={:8.3}ms median={:8.3}ms min={:8.3}ms max={:8.3}ms std={:6.3}ms",+ self.name, self.mean_ms, self.median_ms, self.min_ms, self.max_ms, self.std_dev_ms+ );+ }+}++pub fn run_perf<F: FnMut()>(+ name: &str,+ iterations: usize,+ f: F,+) -> PerfResult {+ run_perf_with_warmup(name, 0, iterations, f)+}++pub fn run_perf_with_warmup<F: FnMut()>(+ name: &str,+ warmup: usize,+ iterations: usize,+ mut f: F,+) -> PerfResult {+ // Warmup+ for _ in 0..warmup {+ f();+ }++ // Measure+ let mut times_ms = Vec::new();+ for _ in 0..iterations {+ let start = Instant::now();+ f();+ let elapsed = start.elapsed();+ times_ms.push(elapsed.as_secs_f64() * 1000.0);+ }++ PerfResult::new(name.to_string(), ×_ms)+}diff --git a/crates/uzu/tests/integration/parameters/loader_test.rs b/crates/uzu/tests/integration/parameters/loader_test.rsindex a717f1a3b..a2988ba36 100644--- a/crates/uzu/tests/integration/parameters/loader_test.rs+++ b/crates/uzu/tests/integration/parameters/loader_test.rs@@ -12,7 +12,7 @@ use uzu::{},};-use crate::util::path::get_test_weights_path;+use crate::common::path::get_test_weights_path;#[tag(heavy)]#[test]diff --git a/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rs b/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rsindex 23a4ef027..b7b85bf48 100644--- a/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rs+++ b/crates/uzu/tests/integration/parameters/safetensors_metadata_test.rs@@ -2,7 +2,7 @@ use std::fs::File;use uzu::read_safetensors_metadata;-use crate::util::path::get_test_weights_path;+use crate::common::path::get_test_weights_path;#[test]fn test_metadata_loading() {diff --git a/crates/uzu/tests/integration/session/chat_session/context_mode_test.rs b/crates/uzu/tests/integration/session/chat_session/context_mode_test.rsindex af31c1f20..5ae387c0e 100644--- a/crates/uzu/tests/integration/session/chat_session/context_mode_test.rs+++ b/crates/uzu/tests/integration/session/chat_session/context_mode_test.rs@@ -12,7 +12,7 @@ use uzu::session::{types::{Input, Message, Output},};-use crate::util::path::get_test_model_path;+use crate::common::path::get_test_model_path;fn build_model_path() -> PathBuf {get_test_model_path()diff --git a/crates/uzu/tests/integration/session/chat_session/text_session_test.rs b/crates/uzu/tests/integration/session/chat_session/text_session_test.rsindex 97f56b578..180706a65 100644--- a/crates/uzu/tests/integration/session/chat_session/text_session_test.rs+++ b/crates/uzu/tests/integration/session/chat_session/text_session_test.rs@@ -8,7 +8,7 @@ use uzu::session::{types::{Input, Message, Output},};-use crate::util::path::get_test_model_path;+use crate::common::path::get_test_model_path;fn build_decoding_config() -> DecodingConfig {DecodingConfig::default().with_sampling_seed(SamplingSeed::Custom(42))diff --git a/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rs b/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rsindex f8c4095e9..b9f247224 100644--- a/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rs+++ b/crates/uzu/tests/integration/session/config/grammar_json_schema_test.rs@@ -12,7 +12,7 @@ use uzu::session::{types::Input,};-use crate::util::{path::get_test_model_path, speculator::RepeatSpeculator};+use crate::{common::path::get_test_model_path, util::speculator::RepeatSpeculator};#[derive(Debug, Serialize, Deserialize, JsonSchema)]struct Address {diff --git a/crates/uzu/tests/integration/tracer/trace_validator_test.rs b/crates/uzu/tests/integration/tracer/trace_validator_test.rsindex a98bf2cdc..477e46063 100644--- a/crates/uzu/tests/integration/tracer/trace_validator_test.rs+++ b/crates/uzu/tests/integration/tracer/trace_validator_test.rs@@ -2,7 +2,7 @@use uzu::{TraceValidator, backends::metal::Metal};-use crate::util::path::{get_test_model_path, get_traces_path};+use crate::common::path::{get_test_model_path, get_traces_path};#[test]fn test_tracer() {diff --git a/crates/uzu/tests/integration/util/mod.rs b/crates/uzu/tests/integration/util/mod.rsindex a20a93af9..c91484f20 100644--- a/crates/uzu/tests/integration/util/mod.rs+++ b/crates/uzu/tests/integration/util/mod.rs@@ -1,4 +1,2 @@-#[cfg_attr(not(feature = "tracing"), allow(dead_code))]-pub mod path;#[cfg(metal_backend)]pub mod speculator;diff --git a/crates/uzu/tests/performance/mod.rs b/crates/uzu/tests/performance/mod.rsindex 01ae6f730..9e2de76d1 100644--- a/crates/uzu/tests/performance/mod.rs+++ b/crates/uzu/tests/performance/mod.rs@@ -1,2 +1,7 @@+#[macro_use]+#[path = "../common/mod.rs"]+mod common;+#[cfg(metal_backend)]mod matmul;+mod model_loading_perf_test;diff --git a/crates/uzu/tests/performance/model_loading_perf_test.rs b/crates/uzu/tests/performance/model_loading_perf_test.rsnew file mode 100644index 000000000..fc1645955--- /dev/null+++ b/crates/uzu/tests/performance/model_loading_perf_test.rs@@ -0,0 +1,13 @@+use uzu::prelude::{ChatSession, DecodingConfig};++use crate::common::{path::get_test_model_path, perf::run_perf};++#[test]+#[ignore]+fn test_perf_model_loading() {+ let perf_results = run_perf("Model loading", 10, || {+ let model_path = get_test_model_path();+ let _ = ChatSession::new(model_path, DecodingConfig::default());+ });+ perf_results.print();+}diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rsindex b52980d71..eb4f77a06 100644--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs@@ -45,8 +45,8 @@ fn make_test_layer(};let shape = [1, total_len.max(1), 1];- let keys = context.create_array(&shape, DataType::F32, "kv_cache_keys");- let values = context.create_array(&shape, DataType::F32, "kv_cache_values");+ let keys = context.create_array_zeros(&shape, DataType::F32, "kv_cache_keys");+ let values = context.create_array_zeros(&shape, DataType::F32, "kv_cache_values");let prefix_token_positions = match &state {KVCacheLayerState::Full {diff --git a/crates/uzu/tests/unit/kernel/audio/audio_test.rs b/crates/uzu/tests/unit/kernel/audio/audio_test.rsindex fc876cf92..e53ead660 100644--- a/crates/uzu/tests/unit/kernel/audio/audio_test.rs+++ b/crates/uzu/tests/unit/kernel/audio/audio_test.rs@@ -80,19 +80,19 @@ fn audio_conv1d_replicate_matches_reference_f32() {let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07 - 1.3).cos() * 0.2).collect();let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.01 - 0.02).collect();- let mut input = context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array(&[cout], DataType::F32, "audio_conv1d_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_conv1d_bias");bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_conv1d_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_conv1d_lengths");lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);- let output = context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");+ let output = context.create_array_zeros(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");let expected = conv1d_reference(Conv1dSpec {input: &input_values,weight: &weight_values,@@ -173,19 +173,19 @@ fn audio_causal_conv1d_matches_reference_f32() {let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.02 - 0.5).cos() * 0.1).collect();let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.001 - 0.002).collect();- let mut input = context.create_array(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");+ let mut input = context.create_array_zeros(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");+ let mut weight = context.create_array_zeros(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv1d_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv1d_bias");bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");+ let mut lengths_array = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);- let output = context.create_array(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");+ let output = context.create_array_zeros(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");let expected = causal_conv1d_reference(CausalConv1dSpec {input: &input_values,weight: &weight_values,@@ -263,21 +263,27 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {let bias_values: Vec<f32> = vec![0.01, -0.02];let mut input =- context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");+ context.create_array_zeros(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut weight =- context.create_array(&[cout, cin / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");+ let mut weight = context.create_array_zeros(+ &[cout, cin / groups, kernel_size],+ DataType::F32,+ "audio_causal_conv_transpose_weight",+ );weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);- let output =- context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_causal_conv_transpose_output");+ let output = context.create_array_zeros(+ &[batch_size, cout, seq_len_out],+ DataType::F32,+ "audio_causal_conv_transpose_output",+ );let expected = causal_conv_transpose1d_reference(CausalConvTranspose1dSpec {input: &input_values,weight: &weight_values,@@ -385,24 +391,25 @@ fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {("NCS", 0_i32, [batch_size, cin, seq_len_in], input_values_ncs.as_slice()),("NSC", 1_i32, [batch_size, seq_len_in, cin], input_values_nsc.as_slice()),] {- let mut input = context.create_array(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");+ let mut input =+ context.create_array_zeros(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");input.as_slice_mut::<f32>().copy_from_slice(input_values);- let mut weight = context.create_array(+ let mut weight = context.create_array_zeros(&[cout, cin / groups, kernel_size],DataType::F32,"audio_causal_conv_transpose_lalamo_weight",);weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");+ let mut bias = context.create_array_zeros(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);let mut lengths =- context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");+ context.create_array_zeros(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);- let output = context.create_array(+ let output = context.create_array_zeros(&[batch_size, cout, seq_len_out],DataType::F32,"audio_causal_conv_transpose_lalamo_output",@@ -457,11 +464,11 @@ fn audio_leaky_relu_matches_reference_f32() {let eps = 1e-6f32;let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();- let mut input = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");+ let mut input = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let alpha = context.create_array(&[1], DataType::F32, "audio_leaky_relu_alpha");- let output = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");+ let alpha = context.create_array_zeros(&[1], DataType::F32, "audio_leaky_relu_alpha");+ let output = context.create_array_zeros(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);@@ -502,10 +509,10 @@ fn audio_tanh_matches_reference_f32() {let n = 1024usize;let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();- let mut input = context.create_array(&[n], DataType::F32, "audio_tanh_input");+ let mut input = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");+ let output = context.create_array_zeros(&[n], DataType::F32, "audio_tanh_output");run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);@@ -531,13 +538,13 @@ fn audio_add_matches_reference_f32() {let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();- let mut a = context.create_array(&[n], DataType::F32, "audio_add_a");+ let mut a = context.create_array_zeros(&[n], DataType::F32, "audio_add_a");a.as_slice_mut::<f32>().copy_from_slice(&a_values);- let mut b = context.create_array(&[n], DataType::F32, "audio_add_b");+ let mut b = context.create_array_zeros(&[n], DataType::F32, "audio_add_b");b.as_slice_mut::<f32>().copy_from_slice(&b_values);- let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");+ let sum = context.create_array_zeros(&[n], DataType::F32, "audio_add_sum");run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(a_buffer = a);@@ -578,13 +585,14 @@ fn audio_half_snake_matches_reference_f32() {let input_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.01 - 3.2).sin()).collect();let alpha_values: Vec<f32> = (0..snake_channels).map(|i| 0.5 + i as f32 * 0.1).collect();- let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");+ let mut input =+ context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut alpha = context.create_array(&[snake_channels], DataType::F32, "audio_half_snake_alpha");+ let mut alpha = context.create_array_zeros(&[snake_channels], DataType::F32, "audio_half_snake_alpha");alpha.as_slice_mut::<f32>().copy_from_slice(&alpha_values);- let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");+ let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);@@ -683,19 +691,19 @@ fn audio_norm_ncs_matches_reference_f32() {};for subtract_mean in [false, true] {- let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");+ let mut input = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut scales = context.create_array(&[channels], DataType::F32, "audio_norm_scales");+ let mut scales = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_scales");scales.as_slice_mut::<f32>().copy_from_slice(&scales_values);- let mut bias = context.create_array(&[channels], DataType::F32, "audio_norm_bias");+ let mut bias = context.create_array_zeros(&[channels], DataType::F32, "audio_norm_bias");bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_norm_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_norm_lengths");lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);- let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");+ let output = context.create_array_zeros(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);@@ -749,17 +757,20 @@ fn audio_fsq_decode_matches_reference() {let num_levels: Vec<i32> = vec![8, 5];let dim_base_index: Vec<i32> = vec![1, 8];- let mut tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");+ let mut tokens = context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");tokens.as_slice_mut::<i32>().copy_from_slice(&[0, 7, 11, // g0 tokens at t0..t23, 5, 9, // g1 tokens at t0..t2]);- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_lengths");lengths.as_slice_mut::<i32>().copy_from_slice(&[2]);- let output =- context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");+ let output = context.create_array_zeros(+ &[batch_size, num_groups * codebook_dim, seq_len],+ DataType::F32,+ "audio_fsq_output",+ );run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(tokens_buffer = tokens);@@ -812,17 +823,18 @@ fn audio_fsq_encode_matches_reference() {];let lengths_values = vec![3_i32];- let mut input = context.create_array(+ let mut input = context.create_array_zeros(&[batch_size, num_groups * codebook_dim, seq_len],DataType::F32,"audio_fsq_encode_input",);input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");+ let mut lengths = context.create_array_zeros(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);- let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");+ let tokens =+ context.create_array_zeros(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");run_command_buffer(&context, |command_buffer| {borrow_array_buffer!(input_buffer = input);diff --git a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rsindex 01a5f181f..c64580a5b 100644--- a/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs+++ b/crates/uzu/tests/unit/kernel/moe/moe_perf_test.rs@@ -1,7 +1,5 @@#![cfg(metal_backend)]-use std::time::Instant;-use half::bf16;use rand::{RngExt, SeedableRng, rngs::StdRng};use uzu::{@@ -20,75 +18,7 @@ use uzu::{};use super::test_utils::{alloc_buffer, alloc_buffer_with_data, create_ctx};--struct PerfResult {- name: String,- mean_ms: f64,- median_ms: f64,- min_ms: f64,- max_ms: f64,- std_dev_ms: f64,-}--impl PerfResult {- fn new(- name: String,- times_ms: &[f64],- ) -> Self {- let mut sorted = times_ms.to_vec();- sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());-- let mean = sorted.iter().sum::<f64>() / sorted.len() as f64;- let median = sorted[sorted.len() / 2];- let min = sorted[0];- let max = sorted[sorted.len() - 1];-- let variance = sorted.iter().map(|t| (t - mean).powi(2)).sum::<f64>() / sorted.len() as f64;- let std_dev = variance.sqrt();-- Self {- name,- mean_ms: mean,- median_ms: median,- min_ms: min,- max_ms: max,- std_dev_ms: std_dev,- }- }-- fn print(&self) {- eprintln!(- " {:<30} mean={:8.3}ms median={:8.3}ms min={:8.3}ms max={:8.3}ms std={:6.3}ms",- self.name, self.mean_ms, self.median_ms, self.min_ms, self.max_ms, self.std_dev_ms- );- }-}--fn time_kernel<F>(- name: &str,- warmup: usize,- iterations: usize,- mut f: F,-) -> PerfResult-where- F: FnMut(),-{- // Warmup- for _ in 0..warmup {- f();- }-- // Measure- let mut times_ms = Vec::new();- for _ in 0..iterations {- let start = Instant::now();- f();- let elapsed = start.elapsed();- times_ms.push(elapsed.as_secs_f64() * 1000.0);- }-- PerfResult::new(name.to_string(), ×_ms)-}+use crate::common::perf::run_perf_with_warmup;// Test E2E MoE performance with timing breakdown (decode mode, T=1)#[test]@@ -121,7 +51,7 @@ fn test_moe_e2e_decode_perf() {.expect("router+topk fused kernel");// Time fused Router+TopK- let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {+ let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");router_topk.encode(&x_buf,@@ -179,7 +109,7 @@ fn test_moe_e2e_prefill_perf() {.expect("router+topk fused kernel");// Time fused Router+TopK- let fused_perf = time_kernel("Router+TopK (FUSED)", 5, 20, || {+ let fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 5, 20, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");router_topk.encode(&x_buf,@@ -305,7 +235,7 @@ fn test_moe_pipeline_breakdown_decode() {.expect("router+topk fused");// Testing: Router + TopK + Counts+Offsets (FUSED)- let router_topk_fused_perf = time_kernel("Router+TopK (FUSED)", 2, 5, || {+ let router_topk_fused_perf = run_perf_with_warmup("Router+TopK (FUSED)", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");router_topk_fused_kernel.encode(&x_buf,@@ -323,7 +253,7 @@ fn test_moe_pipeline_breakdown_decode() {encoder.end_encoding().submit().wait_until_completed().unwrap();});- let counts_offsets_perf = time_kernel("Counts+Offsets (FUSED)", 2, 5, || {+ let counts_offsets_perf = run_perf_with_warmup("Counts+Offsets (FUSED)", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");counts_offsets_kernel.encode(&topk_ids_buf,@@ -338,7 +268,7 @@ fn test_moe_pipeline_breakdown_decode() {encoder.end_encoding().submit().wait_until_completed().unwrap();});- let scatter_perf = time_kernel("Scatter", 2, 5, || {+ let scatter_perf = run_perf_with_warmup("Scatter", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");scatter_bases_kernel.encode(&partials_buf,@@ -369,7 +299,7 @@ fn test_moe_pipeline_breakdown_decode() {encoder.end_encoding().submit().wait_until_completed().unwrap();});- let gather_perf = time_kernel("Gather", 2, 5, || {+ let gather_perf = run_perf_with_warmup("Gather", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");gather_kernel.encode(&mut encoder,@@ -387,7 +317,7 @@ fn test_moe_pipeline_breakdown_decode() {encoder.end_encoding().submit().wait_until_completed().unwrap();});- let experts_perf = time_kernel("Experts (MAIN COMPUTE)", 2, 5, || {+ let experts_perf = run_perf_with_warmup("Experts (MAIN COMPUTE)", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");experts_kernel.encode(&mut encoder,@@ -423,7 +353,7 @@ fn test_moe_pipeline_breakdown_decode() {encoder.end_encoding().submit().wait_until_completed().unwrap();});- let finalize_perf = time_kernel("Finalize", 2, 5, || {+ let finalize_perf = run_perf_with_warmup("Finalize", 2, 5, || {let mut encoder = Encoder::new(ctx.as_ref()).expect("Failed to create encoder");finalize_kernel.encode(&tok2row_buf,