A high-performance inference engine for AI models
Remove EncodableBlock trait (#247)
uuuvncc9034396820
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex f19235562..7fa0e433a 100644--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -15,9 +15,13 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[allow(unused)] batch_size: u32,#[allow(unused)] vocab_size: u32,#[allow(unused)] model_dim: u32,- #[allow(unused)] group_size: u32,#[allow(unused)] input_scale: f32,- #[allow(unused)] quant_mode: u32,+ #[allow(unused)]+ #[specialize]+ group_size: u32,+ #[allow(unused)]+ #[specialize]+ quant_mode: u32,) {todo!()}diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 9e0a9225e..74cfc6113 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -18,9 +18,9 @@ KERNEL(QuantizedEmbeddingLookup) (constant uint32_t& batch_size,constant uint32_t& vocab_size,constant uint32_t& model_dim,- constant uint32_t& group_size,constant float& input_scale,- const constant uint& quant_mode,+ const uint32_t group_size SPECIALIZE,+ const uint quant_mode SPECIALIZE,const uint dim_idx AXIS(model_dim, 16),const uint batch_idx AXIS(batch_size, 16)) {@@ -62,4 +62,4 @@ KERNEL(QuantizedEmbeddingLookup) (float out_f = float(scale) * float(quantized_value) + float(bias);out_f *= input_scale;output[thread_position_in_grid] = T(out_f);-}\ No newline at end of file+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rsdeleted file mode 100644index 889e3df5e..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs+++ /dev/null@@ -1 +0,0 @@-// Shader files only — Rust logic has moved to common/kernel/matmul/gemv/diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rsdeleted file mode 100644index b4607a3dd..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs+++ /dev/null@@ -1 +0,0 @@-// Shader files only — Rust logic has moved to common/kernel/matmul/split_k/diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex fc2295746..0dd5bf55a 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -12,7 +12,7 @@ use crate::{backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,},- encodable_block::{EncodableBlock, EncodingParameters},+ encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,};@@ -124,11 +124,11 @@ impl<B: Backend> Classifier<B> {self.context.embed- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode_lookup(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.embedding_norm- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]{@@ -143,7 +143,7 @@ impl<B: Backend> Classifier<B> {}self.context.output_norm- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]{@@ -153,12 +153,12 @@ impl<B: Backend> Classifier<B> {self.context.pooling- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.prediction_head- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;command_buffer@@ -204,11 +204,11 @@ impl<B: Backend> Classifier<B> {self.context.embed- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode_lookup(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.embedding_norm- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;for layer in self.context.layers.iter() {layer@@ -217,15 +217,15 @@ impl<B: Backend> Classifier<B> {}self.context.output_norm- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.pooling- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.prediction_head- .encode(&mut state, &encoding_params, &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;command_bufferdiff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rsindex 3edbfd5d1..2b3a6e4e1 100644--- a/crates/uzu/src/classifier/classifier_context.rs+++ b/crates/uzu/src/classifier/classifier_context.rs@@ -6,8 +6,7 @@ use crate::{classifier::ClassifierError,config::{ClassifierModelConfig, ModelMetadata},encodable_block::{- Activation, ClassifierLayer, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope,- embed_block, linear_block,+ Activation, ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, Rope,},forward_pass::{model_shape::ModelShape,@@ -27,15 +26,15 @@ pub struct ClassifierContext<B: Backend> {pub model_config: ClassifierModelConfig,pub model_shape: ModelShape,- pub embed: Box<dyn EncodableBlock<B>>,+ pub embed: Embedding<B>,pub embedding_norm: Normalization<B>,pub layers: Box<[ClassifierLayer<B>]>,pub output_norm: Normalization<B>,- pub global_rope: Rc<Box<dyn EncodableBlock<B>>>,- pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,+ pub global_rope: Rc<Rope<B>>,+ pub local_rope: Option<Rc<Rope<B>>>,- pub pooling: Box<dyn EncodableBlock<B>>,- pub prediction_head: Box<dyn EncodableBlock<B>>,+ pub pooling: Pooling<B>,+ pub prediction_head: ClassifierPredictionHead<B>,pub sigmoid_kernel: <B::Kernels as Kernels>::SigmoidKernel,}@@ -90,7 +89,14 @@ impl<B: Backend> ClassifierContext<B> {.activation_precision().into();- let embed = embed_block(&decoder_config, context.as_ref(), &root_loader_view);+ let embed = Embedding::new(+ context.as_ref(),+ decoder_config.vocab_size as u32,+ decoder_config.model_dim as u32,+ &decoder_config.embedding_config,+ &root_loader_view.subtree("embedding").expect("Failed to get embedding subtree"),+ )+ .expect("Failed to create embedding");let global_rope = Self::create_rope_block(&context, data_type, RopeType::Global).map_err(Error::Classifier)?;let local_rope = classifier_model_config@@ -192,7 +198,7 @@ impl<B: Backend> ClassifierContext<B> {let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))})?;- let prediction_head_dense = linear_block::<1, B>(+ let prediction_head_dense = <dyn Linear<B>>::new::<1>(&prediction_head_config.dense_config,prediction_head_config.use_dense_bias,model_dim,@@ -203,18 +209,16 @@ impl<B: Backend> ClassifierContext<B> {ArrayId::ClassifierPredictionHeadDense,);- let prediction_head_activation = Box::new(- Activation::<B>::new(- &context,- prediction_head_data_type,- prediction_head_config.activation.clone(),- ArrayId::ClassifierPredictionHeadDense,- ArrayId::ClassifierPredictionHeadDense,- )- .map_err(|e| {- Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))- })?,- );+ let prediction_head_activation = Activation::<B>::new(+ &context,+ prediction_head_data_type,+ prediction_head_config.activation.clone(),+ ArrayId::ClassifierPredictionHeadDense,+ ArrayId::ClassifierPredictionHeadDense,+ )+ .map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))+ })?;let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.norm".to_string()))@@ -234,7 +238,7 @@ impl<B: Backend> ClassifierContext<B> {let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))})?;- let prediction_head_final_linear = linear_block::<1, B>(+ let prediction_head_final_linear = <dyn Linear<B>>::new::<1>(&prediction_head_config.readout_config,true,model_dim,@@ -253,28 +257,26 @@ impl<B: Backend> ClassifierContext<B> {Error::UnableToCreateContext(e.into())})?;- let pooling = Box::new(- Pooling::<B>::new(- context.as_ref(),- data_type,- classifier_model_config.model_config.classifier_pooling.clone(),- model_dim,- )- .map_err(|e| {- eprintln!("Failed to create pooling: {:?}", e);- Error::UnableToCreateContext(e.into())- })?,- );+ let pooling = Pooling::<B>::new(+ context.as_ref(),+ data_type,+ classifier_model_config.model_config.classifier_pooling.clone(),+ model_dim,+ )+ .map_err(|e| {+ eprintln!("Failed to create pooling: {:?}", e);+ Error::UnableToCreateContext(e.into())+ })?;- let prediction_head = Box::new(ClassifierPredictionHead::new(+ let prediction_head = ClassifierPredictionHead::new(prediction_head_dense.map_err(|e| {Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))})?,prediction_head_activation,- Box::new(prediction_head_norm),+ prediction_head_norm,prediction_head_final_linear,num_labels,- ));+ );Ok(Self {context,@@ -298,11 +300,9 @@ impl<B: Backend> ClassifierContext<B> {context: &B::Context,data_type: DataType,rope_type: RopeType,- ) -> Result<Rc<Box<dyn EncodableBlock<B>>>, ClassifierError> {- let rotation: Box<dyn EncodableBlock<B>> = Box::new(- Rope::<B>::new(context, data_type, rope_type)- .map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?,- );+ ) -> Result<Rc<Rope<B>>, ClassifierError> {+ let rotation = Rope::<B>::new(context, data_type, rope_type)+ .map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?;Ok(Rc::new(rotation))}}diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rsindex b9ea318f2..a12e4ec80 100644--- a/crates/uzu/src/config/decoder.rs+++ b/crates/uzu/src/config/decoder.rs@@ -244,9 +244,10 @@ mod tests {let config_str = r#"{"embedding_config": {- "type": "QuantizedTiedEmbeddingConfig",+ "type": "MLXQuantizedTiedEmbeddingConfig","input_scale": null,"logit_soft_cap": null,+ "group_size": 128,"embedding_quantization_mode": "int8","activation_quantization_mode": "int8","activation_precision": "bfloat16"@@ -347,11 +348,12 @@ mod tests {"#;let ground_truth_config = DecoderConfig {- embedding_config: EmbeddingConfig::QuantizedTied {+ embedding_config: EmbeddingConfig::MLXQuantizedTied {common: EmbeddingConfigCommon {input_scale: None,logit_soft_cap: None,},+ group_size: 128,embedding_quantization_mode: QuantizationMode::Int8,activation_quantization_mode: Some(QuantizationMode::Int8),activation_precision: ConfigDataType::BFloat16,diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rsindex 92ce2c57c..9562a22fc 100644--- a/crates/uzu/src/config/embedding.rs+++ b/crates/uzu/src/config/embedding.rs@@ -23,14 +23,6 @@ pub enum EmbeddingConfig {common: EmbeddingConfigCommon,precision: ConfigDataType,},- #[serde(rename = "QuantizedTiedEmbeddingConfig")]- QuantizedTied {- #[serde(flatten)]- common: EmbeddingConfigCommon,- embedding_quantization_mode: QuantizationMode,- activation_quantization_mode: Option<QuantizationMode>,- activation_precision: ConfigDataType,- },#[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]MLXQuantizedTied {#[serde(flatten)]@@ -71,10 +63,6 @@ impl EmbeddingConfig {common,..} => common,- EmbeddingConfig::QuantizedTied {- common,- ..- } => common,EmbeddingConfig::MLXQuantizedTied {common,..@@ -99,30 +87,6 @@ mod tests {#[test]fn test_embedding_config() {- let config_str = r#"- {- "type": "QuantizedTiedEmbeddingConfig",- "input_scale": null,- "logit_soft_cap": null,- "embedding_quantization_mode": "int8",- "activation_quantization_mode": "int8",- "activation_precision": "bfloat16"- }- "#;-- let ground_truth_config = EmbeddingConfig::QuantizedTied {- common: EmbeddingConfigCommon {- input_scale: None,- logit_soft_cap: None,- },- embedding_quantization_mode: QuantizationMode::Int8,- activation_quantization_mode: Some(QuantizationMode::Int8),- activation_precision: ConfigDataType::BFloat16,- };-- let deserialized_config: EmbeddingConfig = from_str(config_str).unwrap();- assert_eq!(deserialized_config, ground_truth_config);-let semi_config_str = r#"{"type": "MLXSemiQuantizedUntiedEmbeddingConfig",diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rsindex 11caea45b..199301464 100644--- a/crates/uzu/src/encodable_block/activation.rs+++ b/crates/uzu/src/encodable_block/activation.rs@@ -2,7 +2,6 @@use std::ops::DerefMut;-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -37,13 +36,10 @@ impl<B: Backend> Activation<B> {output_array_id,})}-}-impl<B: Backend> EncodableBlock<B> for Activation<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex 1a8bf6bf6..e2ab1f4dc 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -14,7 +14,7 @@ use crate::{attention::{AttentionGemmArguments, AttentionGemmBlock},},},- encodable_block::{EncodableBlock, EncodingParameters},+ encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState, HashMapId},};@@ -133,10 +133,8 @@ impl<B: Backend> Attention<B> {KernelVariant::SinglePass}-}-impl<B: Backend> EncodableBlock<B> for Attention<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex 33b3ca455..9c1d60ac5 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -7,8 +7,7 @@ use crate::{backends::common::{Backend, CommandBuffer},config::TransformerLayerConfig,encodable_block::{- Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,- transformer_layer::{linear_block, mlp_block},+ Attention, EncodingParameters, Linear, Mlp, Normalization, QKNorm, Rope, TensorAddSwap, TensorCopy,},forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -17,20 +16,20 @@ use crate::{pub struct ClassifierLayer<B: Backend> {#[cfg_attr(not(feature = "tracing"), allow(dead_code))]layer_index: usize,- copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>>,- pre_attention_norm: Option<Box<dyn EncodableBlock<B>>>,- qkv_projection: Box<dyn EncodableBlock<B>>,- qk_norm: Option<Box<dyn EncodableBlock<B>>>,- rope: Rc<Box<dyn EncodableBlock<B>>>,- attention: Box<dyn EncodableBlock<B>>,- out_projection: Box<dyn EncodableBlock<B>>,- post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,- mixer_residual_add: Box<dyn EncodableBlock<B>>,- copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>>,- pre_mlp_norm: Box<dyn EncodableBlock<B>>,- mlp: Box<dyn EncodableBlock<B>>,- post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,- mlp_residual_add: Box<dyn EncodableBlock<B>>,+ copy_main_to_shortcut_mixer: TensorCopy<B>,+ pre_attention_norm: Option<Normalization<B>>,+ qkv_projection: Box<dyn Linear<B>>,+ qk_norm: Option<QKNorm<B>>,+ rope: Rc<Rope<B>>,+ attention: Attention<B>,+ out_projection: Box<dyn Linear<B>>,+ post_attention_norm: Option<Normalization<B>>,+ mixer_residual_add: TensorAddSwap<B>,+ copy_main_to_shortcut_mlp: TensorCopy<B>,+ pre_mlp_norm: Normalization<B>,+ mlp: Box<dyn Mlp<B>>,+ post_mlp_norm: Option<Normalization<B>>,+ mlp_residual_add: TensorAddSwap<B>,}impl<B: Backend> ClassifierLayer<B> {@@ -45,7 +44,7 @@ impl<B: Backend> ClassifierLayer<B> {num_groups: usize,attention_scale: Option<f32>,layer_loader: &ParameterTree<B::Context>,- rope: Rc<Box<dyn EncodableBlock<B>>>,+ rope: Rc<Rope<B>>,) -> Self {autoreleasepool(|_| {let ctx = context.as_ref(); // Reference for functions expecting &B::Context@@ -53,37 +52,34 @@ impl<B: Backend> ClassifierLayer<B> {layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();- let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>> = Box::new(- TensorCopy::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap(),- );+ let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap();- let pre_attention_norm: Option<Box<dyn EncodableBlock<B>>> =- if let Some(norm_config) = &layer_config.pre_attention_norm_config {- if layer_loader.subtree("pre_mixer_norm").is_ok() {- Some(Box::new(- Normalization::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &layer_loader.subtree("pre_mixer_norm").unwrap(),- )- .expect("Failed to create pre-attention norm kernel"),- ))- } else {- None- }+ let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {+ if layer_loader.subtree("pre_mixer_norm").is_ok() {+ Some(+ Normalization::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("pre_mixer_norm").unwrap(),+ )+ .expect("Failed to create pre-attention norm kernel"),+ )} else {None- };+ }+ } else {+ None+ };- let qkv_projection = linear_block(+ let qkv_projection = <dyn Linear<B>>::new(&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,@@ -95,27 +91,27 @@ impl<B: Backend> ClassifierLayer<B> {).expect("Failed to create qkv projection");- let qk_norm: Option<Box<dyn EncodableBlock<B>>> =- if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {- match QKNorm::new(- ctx,- intermediate_data_type,- attention_config.query_norm_config.clone(),- attention_config.key_norm_config.clone(),- ArrayId::QKV,- &layer_loader.subtree("mixer").unwrap(),- num_heads,- num_groups,- head_dim,- ) {- Ok(norm) => Some(Box::new(norm)),- Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),- }- } else {- None- };+ let qk_norm = if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some()+ {+ match QKNorm::new(+ ctx,+ intermediate_data_type,+ attention_config.query_norm_config.clone(),+ attention_config.key_norm_config.clone(),+ ArrayId::QKV,+ &layer_loader.subtree("mixer").unwrap(),+ num_heads,+ num_groups,+ head_dim,+ ) {+ Ok(norm) => Some(norm),+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),+ }+ } else {+ None+ };- let out_projection = linear_block(+ let out_projection = <dyn Linear<B>>::new(&attention_config.out_projection_config,attention_config.has_out_biases,num_heads * head_dim,@@ -127,54 +123,47 @@ impl<B: Backend> ClassifierLayer<B> {).expect("Failed to create out projection");- let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =- if let Some(norm_config) = &layer_config.post_attention_norm_config {- Some(Box::new(- Normalization::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &layer_loader.subtree("post_mixer_norm").unwrap(),- )- .expect("Failed to create post-attention norm kernel"),- ))- } else {- None- };-- let mixer_residual_add: Box<dyn EncodableBlock<B>> = Box::new(- TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {+ Some(+ Normalization::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("post_mixer_norm").unwrap(),+ )+ .expect("Failed to create post-attention norm kernel"),)- .unwrap(),- );+ } else {+ None+ };- let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>> = Box::new(- TensorCopy::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap(),- );+ let mixer_residual_add = TensorAddSwap::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap();- let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(- Normalization::new(- ctx,- intermediate_data_type,- layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &layer_loader.subtree("pre_mlp_norm").unwrap(),- )- .expect("Failed to create pre-MLP norm kernel"),- );+ let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap();++ let pre_mlp_norm = Normalization::new(+ ctx,+ intermediate_data_type,+ layer_config.pre_mlp_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("pre_mlp_norm").unwrap(),+ )+ .expect("Failed to create pre-MLP norm kernel");- let mlp = mlp_block(+ let mlp = <dyn Mlp<B>>::new(&layer_config.mlp_config,model_dim,hidden_dim,@@ -183,44 +172,39 @@ impl<B: Backend> ClassifierLayer<B> {).expect("Failed to create mlp block");- let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =- if let Some(norm_config) = &layer_config.post_mlp_norm_config {- Some(Box::new(- Normalization::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &layer_loader.subtree("post_mlp_norm").unwrap(),- )- .expect("Failed to create post-MLP norm kernel"),- ))- } else {- None- };-- let attention: Box<dyn EncodableBlock<B>> = Box::new(- Attention::new(- ctx,- intermediate_data_type,- layer_index,- attention_scale,- attention_config.has_sinks,- false,- attention_config.sliding_window_size,+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ Some(+ Normalization::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &layer_loader.subtree("post_mlp_norm").unwrap(),+ )+ .expect("Failed to create post-MLP norm kernel"),)- .expect("Failed to create attention kernel"),- );+ } else {+ None+ };- let mlp_residual_add: Box<dyn EncodableBlock<B>> = Box::new(- TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),- )- .unwrap(),- );+ let attention = Attention::new(+ ctx,+ intermediate_data_type,+ layer_index,+ attention_scale,+ attention_config.has_sinks,+ false,+ attention_config.sliding_window_size,+ )+ .expect("Failed to create attention kernel");++ let mlp_residual_add = TensorAddSwap::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap();Self {layer_index,@@ -241,10 +225,8 @@ impl<B: Backend> ClassifierLayer<B> {}})}-}-impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,@@ -258,10 +240,10 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());}- self.copy_main_to_shortcut_mixer.encode(state, parameters, command_buffer)?;+ self.copy_main_to_shortcut_mixer.encode(state, command_buffer)?;if let Some(ref pre_attn_norm) = self.pre_attention_norm {- pre_attn_norm.encode(state, parameters, command_buffer)?;+ pre_attn_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(@@ -281,20 +263,20 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {}}- self.qkv_projection.encode(state, parameters, command_buffer)?;+ self.qkv_projection.encode(state, command_buffer)?;if let Some(ref qk_norm) = self.qk_norm {- qk_norm.encode(state, parameters, command_buffer)?;+ qk_norm.encode(state, command_buffer)?;}- self.rope.encode(state, parameters, command_buffer)?;+ self.rope.encode(state, command_buffer)?;self.attention.encode(state, parameters, command_buffer)?;- self.out_projection.encode(state, parameters, command_buffer)?;+ self.out_projection.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());}if let Some(ref post_attn_norm) = self.post_attention_norm {- post_attn_norm.encode(state, parameters, command_buffer)?;+ post_attn_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(@@ -305,41 +287,40 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {}}- self.mixer_residual_add.encode(state, parameters, command_buffer)?;+ self.mixer_residual_add.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());}- self.copy_main_to_shortcut_mlp.encode(state, parameters, command_buffer)?;+ self.copy_main_to_shortcut_mlp.encode(state, command_buffer)?;- self.pre_mlp_norm.encode(state, parameters, command_buffer)?;+ self.pre_mlp_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());}- self.mlp.encode(state, parameters, command_buffer)?;+ self.mlp.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());}if let Some(ref post_mlp_norm) = self.post_mlp_norm {- post_mlp_norm.encode(state, parameters, command_buffer)?;+ post_mlp_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());}}- self.mlp_residual_add.encode(state, parameters, command_buffer)?;+ self.mlp_residual_add.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());}- let _ = parameters;Ok(())}}diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex 75b4338c7..890b474a9 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -2,14 +2,13 @@use std::rc::Rc;+use thiserror::Error;+use crate::{DataType, DecoderConfig,backends::common::{Backend, CommandBuffer},config::{DecoderLayerType, MixerConfig},- encodable_block::{- EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,- transformer_layer::{embed_block, readout_block},- },+ encodable_block::{Embedding, EncodingParameters, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError},forward_pass::{model_shape::ModelShape,state::{ArrayId, ForwardPassState, RopeType},@@ -17,14 +16,21 @@ use crate::{parameters::ParameterTree,};+#[derive(Debug, Error)]+pub enum DecoderError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Embedding error: {0}")]+ EmbeddingError(#[from] EmbeddingError<B>),+}+/// Full decoder executable with all layers and components.pub struct Decoder<B: Backend> {- pub embed: Box<dyn EncodableBlock<B>>,+ pub embed: Embedding<B>,pub layers: Box<[LayerExecutables<B>]>,- pub norm: Box<dyn EncodableBlock<B>>,- pub readout: Box<dyn EncodableBlock<B>>,- pub global_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,- pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,+ pub norm: RMSNorm<B>,+ pub global_rope: Option<Rc<Rope<B>>>,+ pub local_rope: Option<Rc<Rope<B>>>,}impl<B: Backend> Decoder<B> {@@ -35,9 +41,14 @@ impl<B: Backend> Decoder<B> {) -> Self {let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");- let embed = embed_block(&decoder_config, context.as_ref(), root_weight_loader);-- let readout = readout_block(&decoder_config, context.as_ref(), root_weight_loader);+ let embed = Embedding::new(+ context.as_ref(),+ decoder_config.vocab_size as u32,+ decoder_config.model_dim as u32,+ &decoder_config.embedding_config,+ &root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree"),+ )+ .expect("Failed to create embedding");let attention_data_type = Self::attention_data_type(&decoder_config);let norm_reference_layer =@@ -114,24 +125,21 @@ impl<B: Backend> Decoder<B> {}).collect::<Vec<_>>();- let norm_block: Box<dyn EncodableBlock<B>> = Box::new(- RMSNorm::new(- context.as_ref(),- norm_data_type,- decoder_config.output_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_weight_loader.subtree("output_norm").unwrap(),- )- .map(RMSNorm::with_sampling_range)- .expect("Failed to create output RMS norm kernel"),- );+ let norm_block = RMSNorm::new(+ context.as_ref(),+ norm_data_type,+ decoder_config.output_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_weight_loader.subtree("output_norm").unwrap(),+ )+ .map(RMSNorm::with_sampling_range)+ .expect("Failed to create output RMS norm kernel");Self {embed,layers: layers.into_boxed_slice(),norm: norm_block,- readout,global_rope,local_rope,}@@ -141,10 +149,8 @@ impl<B: Backend> Decoder<B> {context: &B::Context,data_type: DataType,rope_type: RopeType,- ) -> Rc<Box<dyn EncodableBlock<B>>> {- let rotation: Box<dyn EncodableBlock<B>> =- Box::new(Rope::<B>::new(context, data_type, rope_type).expect("Failed to create Rope"));- Rc::new(rotation)+ ) -> Rc<Rope<B>> {+ Rc::new(Rope::<B>::new(context, data_type, rope_type).expect("Failed to create Rope"))}fn attention_data_type(decoder_config: &DecoderConfig) -> Option<DataType> {@@ -159,33 +165,31 @@ impl<B: Backend> Decoder<B> {.map(|attention_config| attention_config.qkv_projection_config.activation_precision().into())})}-}-impl<B: Backend> EncodableBlock<B> for Decoder<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- self.embed.encode(state, parameters, command_buffer)?;+ ) -> Result<(), DecoderError<B>> {+ self.embed.encode_lookup(state, command_buffer)?;for layer in self.layers.iter() {- layer.encode(state, parameters, command_buffer)?;+ layer.encode(state, parameters, command_buffer).map_err(DecoderError::BackendError)?;}if state.is_prefilling() {return Ok(());}- self.norm.encode(state, parameters, command_buffer)?;+ self.norm.encode(state, command_buffer).map_err(DecoderError::BackendError)?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();state.encode_copy_array(command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());}- self.readout.encode(state, parameters, command_buffer)?;+ self.embed.encode_readout(state, command_buffer)?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsnew file mode 100644index 000000000..4e6deb496--- /dev/null+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -0,0 +1,611 @@+use std::{+ cell::RefCell,+ ops::{Deref, DerefMut},+};++use thiserror::Error;++use crate::{+ DataType, EmbeddingConfig, QuantizationMode,+ backends::common::{+ Backend, CommandBuffer, Kernels,+ kernel::{+ FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,+ matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},+ quant_matmul::{+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,+ QuantizedMatmulKernelEncodable, QuantizedMatmulType,+ },+ },+ },+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLeaf, ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum EmbeddingError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Matmul error: {0}")]+ MatmulError(#[from] MatmulError<B>),+ #[error("QuantizedMatmul error: {0}")]+ QuantizedMatmulError(#[from] QuantizedMatmulError<B>),+ #[error("Parameter loading error: {0}")]+ ParameterError(#[from] ParameterLoaderError<B>),+ #[error("Unsupported configuration: {0}")]+ UnsupportedConfiguration(String),+ #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]+ InvalidTensor {+ shape: Box<[usize]>,+ data_type: DataType,+ expected_shape: Box<[usize]>,+ expected_data_type: DataType,+ },+}++enum TiedEmbeddingType<B: Backend> {+ FullPrecision {+ weights: B::Buffer,+ lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,+ readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,+ },+ Quantized {+ weights: B::Buffer,+ scales: B::Buffer,+ biases: B::Buffer,+ lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,+ readout: QuantizedMatmulKernelEncodable<B>,+ },+}++enum UntiedEmbeddingLookupType<B: Backend> {+ FullPrecision {+ weights: B::Buffer,+ lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,+ },+ Quantized {+ weights: B::Buffer,+ scales: B::Buffer,+ biases: B::Buffer,+ lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,+ },+}++enum UntiedEmbeddingReadoutType<B: Backend> {+ FullPrecision {+ weights: B::Buffer,+ readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,+ },+ Quantized {+ weights: B::Buffer,+ scales: B::Buffer,+ biases: B::Buffer,+ readout: QuantizedMatmulKernelEncodable<B>,+ },+}++enum EmbeddingTying<B: Backend> {+ Tied {+ ty: TiedEmbeddingType<B>,+ },+ Untied {+ input_ty: UntiedEmbeddingLookupType<B>,+ output_ty: UntiedEmbeddingReadoutType<B>,+ },+}++pub struct Embedding<B: Backend> {+ tying: EmbeddingTying<B>,+ input_scale: f32,+ vocab_size: u32,+ model_dim: u32,+}++fn quant_mode_to_int(quant_mode: QuantizationMode) -> u32 {+ match quant_mode {+ QuantizationMode::UInt4 => 0,+ QuantizationMode::Int8 => 1,+ QuantizationMode::UInt8 => 2,+ }+}++fn validate_tensor<'file, 'context, 'leaf, B: Backend>(+ weights_leaf: &ParameterLeaf<'file, 'context, 'leaf, B::Context>,+ expected_shape: [usize; 2],+ expected_data_type: DataType,+) -> Result<(), EmbeddingError<B>> {+ let shape = weights_leaf.shape();+ let data_type = weights_leaf.data_type();++ if (shape, data_type) != (expected_shape.as_ref(), expected_data_type) {+ return Err(EmbeddingError::InvalidTensor {+ shape: shape.into(),+ data_type: weights_leaf.data_type(),+ expected_shape: expected_shape.into(),+ expected_data_type,+ });+ }++ Ok(())+}++impl<B: Backend> Embedding<B> {+ pub fn new(+ context: &B::Context,+ vocab_size: u32,+ model_dim: u32,+ config: &EmbeddingConfig,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, EmbeddingError<B>> {+ let common = config.common();++ let tying = match config {+ EmbeddingConfig::Tied {+ common: _,+ precision,+ } => {+ let data_type = (*precision).into();++ let weights_leaf = parameter_tree.leaf("weights")?;+ validate_tensor(&weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;++ let weights = weights_leaf.read_buffer()?;++ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?;+ let readout =+ RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);++ EmbeddingTying::Tied {+ ty: TiedEmbeddingType::FullPrecision {+ weights,+ lookup,+ readout,+ },+ }+ },+ EmbeddingConfig::Untied {+ common: _,+ precision,+ } => {+ let data_type = (*precision).into();++ let input_weights_leaf = parameter_tree.leaf("input_weights")?;+ validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;+ let output_weights_leaf = parameter_tree.leaf("output_weights")?;+ validate_tensor(&output_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;++ let input_weights = input_weights_leaf.read_buffer()?;+ let output_weights = output_weights_leaf.read_buffer()?;++ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?;+ let readout =+ RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);++ EmbeddingTying::Untied {+ input_ty: UntiedEmbeddingLookupType::FullPrecision {+ weights: input_weights,+ lookup,+ },+ output_ty: UntiedEmbeddingReadoutType::FullPrecision {+ weights: output_weights,+ readout,+ },+ }+ },+ EmbeddingConfig::MLXQuantizedTied {+ common: _,+ group_size,+ embedding_quantization_mode,+ activation_quantization_mode,+ activation_precision,+ } => {+ let data_type = (*activation_precision).into();++ let packing_divisor = embedding_quantization_mode.packing_divisor();+ let storage_data_type = embedding_quantization_mode.storage_type();++ let num_groups = (model_dim as usize).div_ceil(*group_size);++ let weights_leaf = parameter_tree.leaf("weights")?;+ validate_tensor(+ &weights_leaf,+ [vocab_size as usize, model_dim as usize / packing_divisor],+ storage_data_type,+ )?;+ let scales_leaf = parameter_tree.leaf("scales")?;+ validate_tensor(&scales_leaf, [vocab_size as usize, num_groups], data_type)?;+ let biases_leaf = parameter_tree.leaf("biases")?;+ validate_tensor(&biases_leaf, [vocab_size as usize, num_groups], data_type)?;++ let weights = weights_leaf.read_buffer()?;+ let scales = scales_leaf.read_buffer()?;+ let biases = biases_leaf.read_buffer()?;++ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ context,+ data_type,+ *group_size as u32,+ quant_mode_to_int(*embedding_quantization_mode),+ )+ .map_err(EmbeddingError::BackendError)?;+ let readout = QuantizedMatmulKernelEncodable::new(+ context,+ QuantizedMatmulConfiguration {+ data_type,+ group_size: *group_size,+ input_dim: model_dim as usize,+ output_dim: vocab_size as usize,+ mode: *embedding_quantization_mode,+ quantization_type: QuantizedMatmulType::Mlx,+ weights_transposed: true,+ },+ )?;++ if let Some(activation_quantization_mode) = activation_quantization_mode {+ return Err(EmbeddingError::UnsupportedConfiguration(format!(+ "activation_quantization_mode={activation_quantization_mode:?}"+ )));+ }++ EmbeddingTying::Tied {+ ty: TiedEmbeddingType::Quantized {+ weights,+ scales,+ biases,+ lookup,+ readout,+ },+ }+ },+ EmbeddingConfig::MLXQuantizedUntied {+ common: _,+ group_size,+ embedding_quantization_mode,+ activation_quantization_mode,+ activation_precision,+ } => {+ let data_type = (*activation_precision).into();++ let packing_divisor = embedding_quantization_mode.packing_divisor();+ let storage_data_type = embedding_quantization_mode.storage_type();++ let num_groups = (model_dim as usize).div_ceil(*group_size);++ let input_weights_leaf = parameter_tree.leaf("input_weights")?;+ validate_tensor(+ &input_weights_leaf,+ [vocab_size as usize, model_dim as usize / packing_divisor],+ storage_data_type,+ )?;+ let input_scales_leaf = parameter_tree.leaf("input_scales")?;+ validate_tensor(&input_scales_leaf, [vocab_size as usize, num_groups], data_type)?;+ let input_biases_leaf = parameter_tree.leaf("input_biases")?;+ validate_tensor(&input_biases_leaf, [vocab_size as usize, num_groups], data_type)?;++ let output_weights_leaf = parameter_tree.leaf("output_weights")?;+ validate_tensor(+ &output_weights_leaf,+ [vocab_size as usize, model_dim as usize / packing_divisor],+ storage_data_type,+ )?;+ let output_scales_leaf = parameter_tree.leaf("output_scales")?;+ validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;+ let output_biases_leaf = parameter_tree.leaf("output_biases")?;+ validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;++ let input_weights = input_weights_leaf.read_buffer()?;+ let input_scales = input_scales_leaf.read_buffer()?;+ let input_biases = input_biases_leaf.read_buffer()?;++ let output_weights = output_weights_leaf.read_buffer()?;+ let output_scales = output_scales_leaf.read_buffer()?;+ let output_biases = output_biases_leaf.read_buffer()?;++ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(+ context,+ data_type,+ *group_size as u32,+ quant_mode_to_int(*embedding_quantization_mode),+ )+ .map_err(EmbeddingError::BackendError)?;+ let readout = QuantizedMatmulKernelEncodable::new(+ context,+ QuantizedMatmulConfiguration {+ data_type,+ group_size: *group_size,+ input_dim: model_dim as usize,+ output_dim: vocab_size as usize,+ mode: *embedding_quantization_mode,+ quantization_type: QuantizedMatmulType::Mlx,+ weights_transposed: true,+ },+ )?;++ if let Some(activation_quantization_mode) = activation_quantization_mode {+ return Err(EmbeddingError::UnsupportedConfiguration(format!(+ "activation_quantization_mode={activation_quantization_mode:?}"+ )));+ }++ EmbeddingTying::Untied {+ input_ty: UntiedEmbeddingLookupType::Quantized {+ weights: input_weights,+ scales: input_scales,+ biases: input_biases,+ lookup,+ },+ output_ty: UntiedEmbeddingReadoutType::Quantized {+ weights: output_weights,+ scales: output_scales,+ biases: output_biases,+ readout,+ },+ }+ },+ EmbeddingConfig::MLXSemiQuantizedUntied {+ common: _,+ group_size,+ embedding_quantization_mode,+ activation_quantization_mode,+ activation_precision,+ } => {+ let data_type = (*activation_precision).into();++ let packing_divisor = embedding_quantization_mode.packing_divisor();+ let storage_data_type = embedding_quantization_mode.storage_type();++ let num_groups = (model_dim as usize).div_ceil(*group_size);++ let input_weights_leaf = parameter_tree.leaf("input_weights")?;+ validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;++ let output_weights_leaf = parameter_tree.leaf("output_weights")?;+ validate_tensor(+ &output_weights_leaf,+ [vocab_size as usize, model_dim as usize / packing_divisor],+ storage_data_type,+ )?;+ let output_scales_leaf = parameter_tree.leaf("output_scales")?;+ validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;+ let output_biases_leaf = parameter_tree.leaf("output_biases")?;+ validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;++ let input_weights = input_weights_leaf.read_buffer()?;++ let output_weights = output_weights_leaf.read_buffer()?;+ let output_scales = output_scales_leaf.read_buffer()?;+ let output_biases = output_biases_leaf.read_buffer()?;++ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)+ .map_err(EmbeddingError::BackendError)?;+ let readout = QuantizedMatmulKernelEncodable::new(+ context,+ QuantizedMatmulConfiguration {+ data_type,+ group_size: *group_size,+ input_dim: model_dim as usize,+ output_dim: vocab_size as usize,+ mode: *embedding_quantization_mode,+ quantization_type: QuantizedMatmulType::Mlx,+ weights_transposed: true,+ },+ )?;++ if let Some(activation_quantization_mode) = activation_quantization_mode {+ return Err(EmbeddingError::UnsupportedConfiguration(format!(+ "activation_quantization_mode={activation_quantization_mode:?}"+ )));+ }++ EmbeddingTying::Untied {+ input_ty: UntiedEmbeddingLookupType::FullPrecision {+ weights: input_weights,+ lookup,+ },+ output_ty: UntiedEmbeddingReadoutType::Quantized {+ weights: output_weights,+ scales: output_scales,+ biases: output_biases,+ readout,+ },+ }+ },+ };++ let input_scale = common.input_scale.unwrap_or(1.0);++ if let Some(logit_soft_cap) = common.logit_soft_cap {+ return Err(EmbeddingError::UnsupportedConfiguration(format!("logit_soft_cap={logit_soft_cap:?}")));+ }++ Ok(Self {+ tying,+ input_scale,+ vocab_size,+ model_dim,+ })+ }++ pub fn encode_lookup(+ &self,+ state: &mut ForwardPassState<B>,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), EmbeddingError<B>> {+ let batch_size = state.active_suffix_length() as u32;++ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);++ let token_ids_array = arrays[0].borrow();+ let token_ids_buffer_rc = token_ids_array.buffer();+ let token_ids_buffer_borrow = token_ids_buffer_rc.borrow();+ let token_ids = token_ids_buffer_borrow.deref();++ let output_array = arrays[1].borrow();+ let output_buffer_rc = output_array.buffer();+ let mut output_buffer_borrow = output_buffer_rc.borrow_mut();+ let output = output_buffer_borrow.deref_mut();++ match &self.tying {+ EmbeddingTying::Tied {+ ty:+ TiedEmbeddingType::FullPrecision {+ weights,+ lookup,+ readout: _,+ },+ }+ | EmbeddingTying::Untied {+ input_ty:+ UntiedEmbeddingLookupType::FullPrecision {+ weights,+ lookup,+ },+ output_ty: _,+ } => lookup.encode(+ token_ids,+ weights,+ output,+ batch_size,+ self.vocab_size,+ self.model_dim,+ self.input_scale,+ command_buffer,+ ),+ EmbeddingTying::Tied {+ ty:+ TiedEmbeddingType::Quantized {+ weights,+ scales,+ biases,+ lookup,+ readout: _,+ },+ }+ | EmbeddingTying::Untied {+ input_ty:+ UntiedEmbeddingLookupType::Quantized {+ weights,+ scales,+ biases,+ lookup,+ },+ output_ty: _,+ } => {+ lookup.encode(+ token_ids,+ weights,+ scales,+ biases,+ output,+ batch_size,+ self.vocab_size,+ self.model_dim,+ self.input_scale,+ command_buffer,+ );+ },+ };++ Ok(())+ }++ pub fn encode_readout(+ &self,+ state: &mut ForwardPassState<B>,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), EmbeddingError<B>> {+ let batch_size = state.sampling_length();++ if batch_size == 0 {+ return Ok(());+ }++ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);++ let input_array = arrays[0].borrow();+ let input_buffer_rc = input_array.buffer();+ let input_buffer_borrow = input_buffer_rc.borrow();+ let input = input_buffer_borrow.deref();++ let output_array = arrays[1].borrow_mut();+ let output_buffer_rc = output_array.buffer();+ let mut output_buffer_borrow = output_buffer_rc.borrow_mut();+ let output = output_buffer_borrow.deref_mut();++ let input_offset = state.sampling_start() * self.model_dim as usize * input_array.data_type().size_in_bytes();++ match &self.tying {+ EmbeddingTying::Tied {+ ty:+ TiedEmbeddingType::FullPrecision {+ weights,+ lookup: _,+ readout,+ },+ }+ | EmbeddingTying::Untied {+ input_ty: _,+ output_ty:+ UntiedEmbeddingReadoutType::FullPrecision {+ weights,+ readout,+ },+ } => {+ readout.borrow_mut().encode(+ state.context(),+ command_buffer,+ FullPrecisionMatmulArguments {+ a: input,+ a_offset: input_offset,+ b: weights,+ output,+ bias: None,+ batch: batch_size,+ input_dim: self.model_dim as usize,+ output_dim: self.vocab_size as usize,+ },+ );+ },+ EmbeddingTying::Tied {+ ty:+ TiedEmbeddingType::Quantized {+ weights,+ scales,+ biases,+ lookup: _,+ readout,+ },+ }+ | EmbeddingTying::Untied {+ input_ty: _,+ output_ty:+ UntiedEmbeddingReadoutType::Quantized {+ weights,+ scales,+ biases,+ readout,+ },+ } => {+ readout.encode(+ command_buffer,+ QuantizedMatmulArguments {+ a_buffer: input,+ a_offset: input_offset,+ b_buffer: weights,+ scales_buffer: scales,+ zero_points_or_biases_buffer: biases,+ output_buffer: output,+ batch: batch_size,+ input_dim: self.model_dim as usize,+ output_dim: self.vocab_size as usize,+ quantization_type: QuantizedMatmulType::Mlx,+ },+ )?;+ },+ };++ Ok(())+ }+}diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rsdeleted file mode 100644index c9e0fc151..000000000--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs+++ /dev/null@@ -1,117 +0,0 @@-//! Full precision embedding lookup encodable.--use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};--use thiserror::Error;--use super::{EncodableBlock, EncodingParameters};-use crate::{- DataType,- backends::common::{- Backend, CommandBuffer,- kernel::{FullPrecisionEmbeddingLookupKernel, Kernels},- },- forward_pass::state::{ArrayId, ForwardPassState},- parameters::{ParameterLoaderError, ParameterTree},-};--#[derive(Debug, Error)]-pub enum FullPrecisionEmbeddingLookupError<B: Backend> {- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),- #[error(- "Embedding lookup weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"- )]- InvalidWeightsShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_model_dim: usize,- },- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidWeightsDataType {- got: DataType,- expected: DataType,- },-}--pub struct FullPrecisionEmbeddingLookup<B: Backend> {- kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,- weights_buffer: Rc<RefCell<B::Buffer>>,- vocab_size: u32,- model_dim: u32,- input_scale: f32,-}--impl<B: Backend> FullPrecisionEmbeddingLookup<B> {- pub fn new(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- input_scale: Option<f32>,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, FullPrecisionEmbeddingLookupError<B>> {- let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)- .map_err(FullPrecisionEmbeddingLookupError::BackendError)?;-- let weights = parameter_tree- .leaf("weights")- .or_else(|_| parameter_tree.leaf("input_weights"))- .map_err(FullPrecisionEmbeddingLookupError::ParameterError)?;-- if weights.shape() != [vocab_size, model_dim] {- return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsShape {- got: weights.shape().to_vec().into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_model_dim: model_dim,- });- }-- if weights.data_type() != data_type {- return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsDataType {- got: weights.data_type(),- expected: data_type,- });- }-- Ok(Self {- kernel,- weights_buffer: weights.buffer(),- vocab_size: vocab_size as u32,- model_dim: model_dim as u32,- input_scale: input_scale.unwrap_or(1.0),- })- }-}--impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {- fn encode(- &self,- state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);- let batch_size = state.active_suffix_length();- let token_ids_array = arrays[0].borrow_mut();- let output_array = arrays[1].borrow_mut();-- self.kernel.encode(- token_ids_array.buffer().borrow().deref(),- self.weights_buffer.borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),- batch_size as u32,- self.vocab_size,- self.model_dim,- self.input_scale,- command_buffer,- );- Ok(())- }-}diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rsdeleted file mode 100644index d993926f0..000000000--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs+++ /dev/null@@ -1,130 +0,0 @@-use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};--use thiserror::Error;--use super::{EncodableBlock, EncodingParameters};-use crate::{- DataType,- backends::common::{- Backend, CommandBuffer,- kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},- },- forward_pass::state::{ArrayId, ForwardPassState},- parameters::{ParameterLoaderError, ParameterTree},-};--#[derive(Debug, Error)]-pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {- #[error("Matmul error: {0}")]- MatmulError(#[from] MatmulError<B>),- #[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),- #[error(- "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"- )]- InvalidWeightsShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_model_dim: usize,- },- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidWeightsDataType {- expected: DataType,- got: DataType,- },-}--pub struct FullPrecisionEmbeddingReadout<B: Backend> {- kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,- weights_buffer: Rc<RefCell<B::Buffer>>,- vocab_size: usize,- model_dim: usize,-}--impl<B: Backend> FullPrecisionEmbeddingReadout<B> {- pub fn new(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, FullPrecisionEmbeddingReadoutError<B>> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(FullPrecisionEmbeddingReadoutError::UnsupportedDataType(data_type));- }-- let weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => {- parameter_tree.leaf("output_weights").map_err(FullPrecisionEmbeddingReadoutError::ParameterError)?- },- };-- let weights_shape = weights.shape().to_vec();- if weights_shape != [vocab_size, model_dim] {- return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsShape {- got: weights_shape.into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_model_dim: model_dim,- });- }-- if weights.data_type() != data_type {- return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsDataType {- expected: data_type,- got: weights.data_type(),- });- }-- let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?;-- Ok(Self {- kernel: RefCell::new(kernel),- weights_buffer: weights.buffer(),- vocab_size,- model_dim,- })- }-}--impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {- fn encode(- &self,- state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);- let batch_size = state.sampling_length();- if batch_size == 0 {- return Ok(());- }-- let sampling_start = state.sampling_start();- let input_array = arrays[0].borrow_mut();- let output_array = arrays[1].borrow_mut();- let element_size = input_array.data_type().size_in_bytes();-- self.kernel.borrow_mut().encode(- state.context(),- command_buffer,- FullPrecisionMatmulArguments {- a: input_array.buffer().borrow().deref(),- a_offset: sampling_start * self.model_dim * element_size,- b: self.weights_buffer.borrow().deref(),- output: output_array.buffer().borrow_mut().deref_mut(),- bias: None,- batch: batch_size,- input_dim: self.model_dim,- output_dim: self.vocab_size,- },- );- Ok(())- }-}diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex b51a5d85c..0c4b4aed5 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -10,9 +10,8 @@ use crate::{backends::common::{Backend, CommandBuffer},config::{DecoderLayerType, MixerConfig},encodable_block::{- Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,+ Attention, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer, TensorAddSwap,TensorCopy,- transformer_layer::{linear_block, mlp_block},},forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -21,14 +20,14 @@ use crate::{/// A single decoder layer with all its components.pub struct LayerExecutables<B: Backend> {pub layer_index: usize,- pub copy_main_to_shortcut: Box<dyn EncodableBlock<B>>,- pub pre_attention_norm: Box<dyn EncodableBlock<B>>,+ pub copy_main_to_shortcut: TensorCopy<B>,+ pub pre_attention_norm: RMSNorm<B>,pub(crate) mixer: MixerExecutables<B>,- pub post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,- pub main_shortcut_add_swap: Box<dyn EncodableBlock<B>>,- pub pre_mlp_norm: Box<dyn EncodableBlock<B>>,- pub mlp: Box<dyn EncodableBlock<B>>,- pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,+ pub post_attention_norm: Option<RMSNorm<B>>,+ pub main_shortcut_add_swap: TensorAddSwap<B>,+ pub pre_mlp_norm: RMSNorm<B>,+ pub mlp: Box<dyn Mlp<B>>,+ pub post_mlp_norm: Option<RMSNorm<B>>,}impl<B: Backend> LayerExecutables<B> {@@ -45,7 +44,7 @@ impl<B: Backend> LayerExecutables<B> {num_groups: usize,attention_scale: Option<f32>,decoder_layer_loader: &ParameterTree<B::Context>,- rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,+ rope: Option<Rc<Rope<B>>>,) -> Self {autoreleasepool(|_| {let ctx = context.as_ref(); // Reference for functions expecting &B::Context@@ -54,32 +53,28 @@ impl<B: Backend> LayerExecutables<B> {MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),};- let copy_main_to_shortcut: Box<dyn EncodableBlock<B>> = Box::new(- TensorCopy::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),- )- .unwrap(),- );+ let copy_main_to_shortcut = TensorCopy::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),+ )+ .unwrap();- let pre_attention_norm: Box<dyn EncodableBlock<B>> = Box::new(- RMSNorm::new(- ctx,- intermediate_data_type,- layer_config.pre_attention_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),- );+ let pre_attention_norm = RMSNorm::new(+ ctx,+ intermediate_data_type,+ layer_config.pre_attention_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel");let mixer = match &layer_config.mixer_config {MixerConfig::Attention(attention_config) => {let rope_block = rope.expect("RoPE encoder missing for attention layer");- let qkv_projection = linear_block(+ let qkv_projection = <dyn Linear<B>>::new(&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,@@ -91,7 +86,7 @@ impl<B: Backend> LayerExecutables<B> {).expect("Failed to create qkv projection");- let qk_norm: Option<Box<dyn EncodableBlock<B>>> =+ let qk_norm =if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {match QKNorm::new(ctx,@@ -104,14 +99,14 @@ impl<B: Backend> LayerExecutables<B> {num_groups,head_dim,) {- Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<B>>),+ Ok(qk_norm) => Some(qk_norm),Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),}} else {None};- let out_projection = linear_block(+ let out_projection = <dyn Linear<B>>::new(&attention_config.out_projection_config,attention_config.has_out_biases,num_heads * head_dim,@@ -123,18 +118,16 @@ impl<B: Backend> LayerExecutables<B> {).expect("Failed to create out projection");- let attention = Box::new(- Attention::new(- ctx,- intermediate_data_type,- layer_index,- attention_scale,- attention_config.has_sinks,- attention_config.is_causal.unwrap_or(true),- attention_config.sliding_window_size,- )- .expect("Failed to create AttentionWrapper kernel"),- );+ let attention = Attention::new(+ ctx,+ intermediate_data_type,+ layer_index,+ attention_scale,+ attention_config.has_sinks,+ attention_config.is_causal.unwrap_or(true),+ attention_config.sliding_window_size,+ )+ .expect("Failed to create AttentionWrapper kernel");MixerExecutables::Attention {qkv_projection,@@ -145,7 +138,7 @@ impl<B: Backend> LayerExecutables<B> {}},MixerConfig::Mamba(mamba_config) => {- let mixer: Box<dyn EncodableBlock<B>> = Box::new(MambaMixer::new(+ let mixer = MambaMixer::new(ctx,layer_type.clone(),mamba_config.clone(),@@ -155,65 +148,60 @@ impl<B: Backend> LayerExecutables<B> {head_dim,num_groups,decoder_layer_loader,- ));+ );MixerExecutables::StateSpace {mixer,}},MixerConfig::ShortConv(short_conv_config) => {- let mixer: Box<dyn EncodableBlock<B>> = Box::new(ShortConvMixer::new(+ let mixer = ShortConvMixer::new(ctx,layer_type.clone(),short_conv_config.clone(),layer_index,model_dim,decoder_layer_loader,- ));+ );MixerExecutables::ShortConv {mixer,}},};- let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =- if let Some(norm_config) = &layer_config.post_attention_norm_config {- Some(Box::new(- RMSNorm::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),- ))- } else {- None- };-- let main_shortcut_add_swap: Box<dyn EncodableBlock<B>> = Box::new(- TensorAddSwap::<B>::new(- ctx,- intermediate_data_type,- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {+ Some(+ RMSNorm::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),)- .unwrap(),- );+ } else {+ None+ };- let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(- RMSNorm::new(- ctx,- intermediate_data_type,- layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),- );+ let main_shortcut_add_swap = TensorAddSwap::<B>::new(+ ctx,+ intermediate_data_type,+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),+ )+ .unwrap();++ let pre_mlp_norm = RMSNorm::new(+ ctx,+ intermediate_data_type,+ layer_config.pre_mlp_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel");- let mlp = mlp_block(+ let mlp = <dyn Mlp<B>>::new(&layer_config.mlp_config,model_dim,hidden_dim,@@ -222,22 +210,21 @@ impl<B: Backend> LayerExecutables<B> {).expect("Failed to create mlp block");- let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =- if let Some(norm_config) = &layer_config.post_mlp_norm_config {- Some(Box::new(- RMSNorm::new(- ctx,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),- ))- } else {- None- };+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {+ Some(+ RMSNorm::new(+ ctx,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),+ )+ } else {+ None+ };Self {layer_index,@@ -252,10 +239,8 @@ impl<B: Backend> LayerExecutables<B> {}})}-}-impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,parameters: &EncodingParameters,@@ -269,10 +254,10 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());}- self.copy_main_to_shortcut.encode(state, parameters, command_buffer)?;+ self.copy_main_to_shortcut.encode(state, command_buffer)?;// shortcut = input- self.pre_attention_norm.encode(state, parameters, command_buffer)?;+ self.pre_attention_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());@@ -286,13 +271,13 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {attention,out_projection,} => {- qkv_projection.encode(state, parameters, command_buffer)?;+ qkv_projection.encode(state, command_buffer)?;if let Some(norm) = qk_norm {- norm.encode(state, parameters, command_buffer)?;+ norm.encode(state, command_buffer)?;}- rope.encode(state, parameters, command_buffer)?;+ rope.encode(state, command_buffer)?;attention.encode(state, parameters, command_buffer)?;- out_projection.encode(state, parameters, command_buffer)?;+ out_projection.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());@@ -301,7 +286,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {MixerExecutables::StateSpace {mixer,} => {- mixer.encode(state, parameters, command_buffer)?;+ mixer.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());@@ -310,7 +295,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {MixerExecutables::ShortConv {mixer,} => {- mixer.encode(state, parameters, command_buffer)?;+ mixer.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());@@ -319,7 +304,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {}if let Some(post_attention_norm) = &self.post_attention_norm {- post_attention_norm.encode(state, parameters, command_buffer)?;+ post_attention_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(@@ -331,7 +316,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {}//main = attention_result- self.main_shortcut_add_swap.encode(state, parameters, command_buffer)?;+ self.main_shortcut_add_swap.encode(state, command_buffer)?;// shortcut = input + attention_result// main = input + attention_result#[cfg(feature = "tracing")]@@ -339,20 +324,20 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());}- self.pre_mlp_norm.encode(state, parameters, command_buffer)?;+ self.pre_mlp_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());}- self.mlp.encode(state, parameters, command_buffer)?;+ self.mlp.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());}if let Some(post_mlp_norm) = &self.post_mlp_norm {- post_mlp_norm.encode(state, parameters, command_buffer)?;+ post_mlp_norm.encode(state, command_buffer)?;#[cfg(feature = "tracing")]if let Some(ref layer_traces) = layer_traces {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());@@ -360,7 +345,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {}// main = mlp_result- self.main_shortcut_add_swap.encode(state, parameters, command_buffer)?;+ self.main_shortcut_add_swap.encode(state, command_buffer)?;// shortcut = input + attention_result + mlp_result// main = input + attention_result + mlp_result#[cfg(feature = "tracing")]@@ -368,7 +353,6 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());}- let _ = parameters;Ok(())}}diff --git a/crates/uzu/src/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rsindex 2ac32fc5d..e709d25b0 100644--- a/crates/uzu/src/encodable_block/layer/mixer.rs+++ b/crates/uzu/src/encodable_block/layer/mixer.rs@@ -2,21 +2,24 @@use std::rc::Rc;-use crate::{backends::common::Backend, encodable_block::EncodableBlock};+use crate::{+ backends::common::Backend,+ encodable_block::{Attention, MambaMixer, QKNorm, Rope, ShortConvMixer, linear::Linear},+};/// Mixer component - either attention, state space, or short conv.pub(crate) enum MixerExecutables<B: Backend> {Attention {- qkv_projection: Box<dyn EncodableBlock<B>>,- qk_norm: Option<Box<dyn EncodableBlock<B>>>,- rope: Rc<Box<dyn EncodableBlock<B>>>,- attention: Box<dyn EncodableBlock<B>>,- out_projection: Box<dyn EncodableBlock<B>>,+ qkv_projection: Box<dyn Linear<B>>,+ qk_norm: Option<QKNorm<B>>,+ rope: Rc<Rope<B>>,+ attention: Attention<B>,+ out_projection: Box<dyn Linear<B>>,},StateSpace {- mixer: Box<dyn EncodableBlock<B>>,+ mixer: MambaMixer<B>,},ShortConv {- mixer: Box<dyn EncodableBlock<B>>,+ mixer: ShortConvMixer<B>,},}diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rsindex 0c2ec7e55..301baf9e6 100644--- a/crates/uzu/src/encodable_block/layer_norm.rs+++ b/crates/uzu/src/encodable_block/layer_norm.rs@@ -8,7 +8,6 @@ use std::{use thiserror::Error;-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -25,7 +24,7 @@ pub enum LayerNormError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),+ ParameterError(ParameterLoaderError<B>),}pub struct LayerNorm<B: Backend> {@@ -45,7 +44,7 @@ impl<B: Backend> LayerNorm<B> {output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,) -> Result<Self, LayerNormError<B>> {- let scales = parameter_tree.leaf("scales").map_err(LayerNormError::ParameterError)?;+ let scales = parameter_tree.leaf_array("scales").map_err(LayerNormError::ParameterError)?;let accumulation_data_type: DataType = config.accumulation_precision.into();let scale_data_type: DataType = config.scale_precision.into();@@ -68,13 +67,10 @@ impl<B: Backend> LayerNorm<B> {scales_buffer: scales.buffer(),})}-}-impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let input_binding = state.arrays(&[self.input_array_id]);diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/linear/full_precision.rssimilarity index 93%rename from crates/uzu/src/encodable_block/full_precision_linear.rsrename to crates/uzu/src/encodable_block/linear/full_precision.rsindex bde5e27ff..2416b1bb2 100644--- a/crates/uzu/src/encodable_block/full_precision_linear.rs+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs@@ -6,7 +6,7 @@ use std::{use thiserror::Error;-use super::{EncodableBlock, EncodingParameters};+use super::Linear;use crate::{DataType,backends::common::{@@ -22,7 +22,7 @@ pub enum FullPrecisionLinearError<B: Backend> {#[error("Matmul error: {0}")]MatmulError(#[from] MatmulError<B>),#[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),+ ParameterError(ParameterLoaderError<B>),#[error("Unsupported data type for full precision linear kernel: {0:?}")]UnsupportedDataType(DataType),#[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]@@ -72,7 +72,7 @@ impl<B: Backend> FullPrecisionLinear<B> {return Err(FullPrecisionLinearError::UnsupportedDataType(precision));}- let weights = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;+ let weights = parameter_tree.leaf_array("weights").map_err(FullPrecisionLinearError::ParameterError)?;let weights_shape = weights.shape().to_vec();if weights_shape != [output_dim, input_dim] {return Err(FullPrecisionLinearError::InvalidWeightsShape {@@ -89,7 +89,7 @@ impl<B: Backend> FullPrecisionLinear<B> {});}- let bias_buffer = match parameter_tree.leaf("biases") {+ let bias_buffer = match parameter_tree.leaf_array("biases") {Ok(biases) => {let bias_shape = biases.shape().to_vec();if bias_shape != [output_dim] {@@ -125,11 +125,10 @@ impl<B: Backend> FullPrecisionLinear<B> {}}-impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {+impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rsnew file mode 100644index 000000000..ed6f482a5--- /dev/null+++ b/crates/uzu/src/encodable_block/linear/mod.rs@@ -0,0 +1,81 @@+mod full_precision;+mod quantized;++pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};+pub use quantized::{QuantizedLinear, QuantizedLinearError};+use thiserror::Error;++use crate::{+ backends::common::{Backend, CommandBuffer},+ config::LinearConfig,+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++pub trait Linear<B: Backend> {+ fn encode(+ &self,+ state: &mut ForwardPassState<B>,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error>;+}++#[derive(Debug, Error)]+pub enum LinearBlockError<B: Backend> {+ #[error("QuantizedLinear error: {0}")]+ QuantizedLinearError(#[source] QuantizedLinearError<B>),+ #[error("FullPrecisionLinear error: {0}")]+ FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[source] ParameterLoaderError<B>),+ #[error("QLoRA linear layer not supported")]+ QLoRaNotSupported,+}++impl<B: Backend> dyn Linear<B> {+ pub fn new<const N: usize>(+ config: &LinearConfig,+ _has_biases: bool,+ input_dimension: usize,+ output_dimensions: [usize; N],+ context: &B::Context,+ parameter_tree: &ParameterTree<B::Context>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {+ let output_dimension_sum: usize = output_dimensions.iter().sum();+ match config {+ LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {+ let block = QuantizedLinear::new(+ context,+ quantization_config,+ input_dimension,+ output_dimension_sum,+ parameter_tree,+ input_array_id,+ output_array_id,+ )+ .map_err(LinearBlockError::QuantizedLinearError)?;+ Ok(Box::new(block))+ },+ LinearConfig::FullPrecision {+ precision,+ } => {+ let block = FullPrecisionLinear::new(+ context,+ (*precision).into(),+ input_dimension,+ output_dimension_sum,+ parameter_tree,+ input_array_id,+ output_array_id,+ )+ .map_err(LinearBlockError::FullPrecisionLinearError)?;+ Ok(Box::new(block))+ },+ LinearConfig::QLoRA {+ ..+ } => Err(LinearBlockError::QLoRaNotSupported),+ }+ }+}diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/linear/quantized.rssimilarity index 94%rename from crates/uzu/src/encodable_block/quantized_linear.rsrename to crates/uzu/src/encodable_block/linear/quantized.rsindex c2fce2e9c..3f46ab886 100644--- a/crates/uzu/src/encodable_block/quantized_linear.rs+++ b/crates/uzu/src/encodable_block/linear/quantized.rs@@ -6,7 +6,7 @@ use std::{use thiserror::Error;-use super::{EncodableBlock, EncodingParameters};+use super::Linear;use crate::{DataType,backends::common::{@@ -31,7 +31,7 @@ pub enum QuantizedLinearError<B: Backend> {#[error("QuantizedMatmul error: {0}")]QuantizedMatmulError(#[source] QuantizedMatmulError<B>),#[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),+ ParameterError(ParameterLoaderError<B>),#[error("Unsupported data type for quantized kernel: {0:?}")]UnsupportedDataType(DataType),#[error("Expected weights of type {expected:?}, got {got:?}")]@@ -114,7 +114,7 @@ impl<B: Backend> QuantizedLinear<B> {return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));}- let weights = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;+ let weights = parameter_tree.leaf_array("weights").map_err(QuantizedLinearError::ParameterError)?;let packing_divisor = config.weight_quantization_mode.packing_divisor();let storage_type = config.weight_quantization_mode.storage_type();if weights.data_type() != storage_type {@@ -124,7 +124,7 @@ impl<B: Backend> QuantizedLinear<B> {});}- let scales = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;+ let scales = parameter_tree.leaf_array("scales").map_err(QuantizedLinearError::ParameterError)?;if scales.data_type() != kernel_data_type {return Err(QuantizedLinearError::InvalidScalesDataType {expected: kernel_data_type,@@ -137,7 +137,7 @@ impl<B: Backend> QuantizedLinear<B> {let scales_shape = scales.shape().to_vec();let weights_transposed = weights_shape[0] == output_dim;- let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf("deq_biases") {+ let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf_array("deq_biases") {Ok(deq_biases) => {let deq_biases_shape = deq_biases.shape().to_vec();if !(weights_shape == [output_dim, input_dim / packing_divisor]@@ -162,7 +162,8 @@ impl<B: Backend> QuantizedLinear<B> {(QuantizedMatmulType::Mlx, deq_biases.buffer())},Err(_) => {- let zero_points = parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;+ let zero_points =+ parameter_tree.leaf_array("zero_points").map_err(QuantizedLinearError::ParameterError)?;let zero_points_shape = zero_points.shape().to_vec();let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;if !(weights_shape == [output_dim, input_dim / packing_divisor]@@ -189,7 +190,7 @@ impl<B: Backend> QuantizedLinear<B> {},};- let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {+ let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf_array("biases") {Ok(biases) => {let bias_shape = biases.shape().to_vec();if bias_shape != [output_dim] {@@ -244,11 +245,10 @@ impl<B: Backend> QuantizedLinear<B> {}}-impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {+impl<B: Backend> Linear<B> for QuantizedLinear<B> {fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex 75948b15e..ff2c8b42a 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -16,7 +16,7 @@ use crate::{},},config::{DecoderLayerType, Mamba2Config},- encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},+ encodable_block::linear::Linear,forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterTree, resolve_subtree},};@@ -24,8 +24,8 @@ use crate::{pub(crate) struct MambaMixer<B: Backend> {layer_index: usize,config: Mamba2Config,- in_projection: Box<dyn EncodableBlock<B>>,- out_projection: Box<dyn EncodableBlock<B>>,+ in_projection: Box<dyn Linear<B>>,+ out_projection: Box<dyn Linear<B>>,split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,conv_decode: <B::Kernels as Kernels>::Conv1dDecodeKernel,conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,@@ -39,6 +39,17 @@ pub(crate) struct MambaMixer<B: Backend> {prefill_mode: SSDPrefillMode,}+fn resolve_prefill_mode_from_env() -> SSDPrefillMode {+ match env::var("UZU_SSM_PREFILL_MODE") {+ Ok(value) => match value.trim().to_ascii_lowercase().as_str() {+ "seq" | "sequential" | "baseline" => SSDPrefillMode::Sequential,+ "single" | "singlepass" | "single_pass" => SSDPrefillMode::SinglePass,+ _ => SSDPrefillMode::SinglePass,+ },+ Err(_) => SSDPrefillMode::SinglePass,+ }+}+impl<B: Backend> MambaMixer<B> {#[allow(clippy::too_many_arguments)]pub(crate) fn new(@@ -61,7 +72,7 @@ impl<B: Backend> MambaMixer<B> {let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();- let in_projection = linear_block(+ let in_projection = <dyn Linear<B>>::new(&mamba_config.in_projection_config,mamba_config.has_in_biases,model_dim,@@ -73,7 +84,7 @@ impl<B: Backend> MambaMixer<B> {).expect("Failed to create in-projection kernel");- let out_projection = linear_block(+ let out_projection = <dyn Linear<B>>::new(&mamba_config.out_projection_config,mamba_config.has_out_biases,mamba_config.inner_dim(),@@ -85,14 +96,14 @@ impl<B: Backend> MambaMixer<B> {).expect("Failed to create out-projection kernel");- let conv_weight = conv_tree.leaf("weights").unwrap();+ let conv_weight = conv_tree.leaf_array("weights").unwrap();let conv_bias = if mamba_config.conv_config.has_biases {- Some(conv_tree.leaf("biases").unwrap())+ Some(conv_tree.leaf_array("biases").unwrap())} else {None};- let gate_bias = split_tree.leaf("gate_bias").unwrap();- let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap();+ let gate_bias = split_tree.leaf_array("gate_bias").unwrap();+ let skip_connection_weight = split_tree.leaf_array("skip_connection_weight").unwrap();let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type).expect("Failed to create split in-projection kernel");@@ -136,9 +147,7 @@ impl<B: Backend> MambaMixer<B> {prefill_mode,}}-}-impl<B: Backend> MambaMixer<B> {fn run_split_inproj(&self,state: &mut ForwardPassState<B>,@@ -397,24 +406,10 @@ impl<B: Backend> MambaMixer<B> {Activation::Gelu => 2,}}-}--fn resolve_prefill_mode_from_env() -> SSDPrefillMode {- match env::var("UZU_SSM_PREFILL_MODE") {- Ok(value) => match value.trim().to_ascii_lowercase().as_str() {- "seq" | "sequential" | "baseline" => SSDPrefillMode::Sequential,- "single" | "singlepass" | "single_pass" => SSDPrefillMode::SinglePass,- _ => SSDPrefillMode::SinglePass,- },- Err(_) => SSDPrefillMode::SinglePass,- }-}-impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {- fn encode(+ pub(crate) fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let active_suffix_length = state.active_suffix_length();@@ -422,7 +417,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {return Ok(());}- self.in_projection.encode(state, parameters, command_buffer)?;+ self.in_projection.encode(state, command_buffer)?;self.run_split_inproj(state, command_buffer, active_suffix_length);self.run_conv_scan(state, command_buffer, active_suffix_length);if active_suffix_length == 1 {@@ -430,7 +425,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {} else {self.run_prefill_ssm(state, command_buffer, active_suffix_length);}- self.out_projection.encode(state, parameters, command_buffer)?;+ self.out_projection.encode(state, command_buffer)?;Ok(())}}diff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp/dense.rssimilarity index 74%rename from crates/uzu/src/encodable_block/mlp.rsrename to crates/uzu/src/encodable_block/mlp/dense.rsindex 36c41f65b..1c2e9eac1 100644--- a/crates/uzu/src/encodable_block/mlp.rs+++ b/crates/uzu/src/encodable_block/mlp/dense.rs@@ -2,23 +2,23 @@use std::ops::{Deref, DerefMut};+use super::{super::linear::Linear, Mlp};use crate::{backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},- encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},};-pub struct MlpBlock<B: Backend> {- up: Box<dyn EncodableBlock<B>>,+pub struct DenseMlp<B: Backend> {+ up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,- down: Box<dyn EncodableBlock<B>>,+ down: Box<dyn Linear<B>>,}-impl<B: Backend> MlpBlock<B> {+impl<B: Backend> DenseMlp<B> {pub fn new(- up: Box<dyn EncodableBlock<B>>,+ up: Box<dyn Linear<B>>,gate: MlpGateActMulEncodable<B>,- down: Box<dyn EncodableBlock<B>>,+ down: Box<dyn Linear<B>>,) -> Self {Self {up,@@ -28,15 +28,14 @@ impl<B: Backend> MlpBlock<B> {}}-impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {+impl<B: Backend> Mlp<B> for DenseMlp<B> {fn encode(&self,state: &mut ForwardPassState<B>,- params: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {// Up- self.up.encode(state, params, command_buffer)?;+ self.up.encode(state, command_buffer)?;// Gate act+mul (fused_up -> hidden)let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);@@ -56,7 +55,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {drop(hidden);// Down- self.down.encode(state, params, command_buffer)?;+ self.down.encode(state, command_buffer)?;Ok(())}}diff --git a/crates/uzu/src/encodable_block/mlp/mod.rs b/crates/uzu/src/encodable_block/mlp/mod.rsnew file mode 100644index 000000000..e6d28a089--- /dev/null+++ b/crates/uzu/src/encodable_block/mlp/mod.rs@@ -0,0 +1,85 @@+mod dense;+mod moe;++pub use dense::DenseMlp;+pub use moe::{MoeBlock, MoeBlockError};+use thiserror::Error;++use super::linear::{Linear, LinearBlockError};+use crate::{+ DataType,+ backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},+ config::MLPConfig,+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++pub trait Mlp<B: Backend> {+ fn encode(+ &self,+ state: &mut ForwardPassState<B>,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<(), B::Error>;+}++#[derive(Debug, Error)]+pub enum MlpBlockError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Linear block error: {0}")]+ LinearBlockError(#[from] LinearBlockError<B>),+ #[error("MoeBlock error: {0}")]+ MoeBlockError(#[from] MoeBlockError<B>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[from] ParameterLoaderError<B>),+}++impl<B: Backend> dyn Mlp<B> {+ pub fn new(+ config: &MLPConfig,+ model_dimension: usize,+ hidden_dimension: usize,+ context: &B::Context,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Box<dyn Mlp<B>>, MlpBlockError<B>> {+ if let MLPConfig::Dense(dense_config) = config {+ let data_type: DataType = dense_config.linear_config.activation_precision().into();++ let up_projection = <dyn Linear<B>>::new(+ &dense_config.linear_config,+ false,+ model_dimension,+ [2 * hidden_dimension],+ context,+ ¶meter_tree.subtree("up_projection")?,+ ArrayId::Main,+ ArrayId::MlpFusedUp,+ )?;++ let gate_activation =+ MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)+ .map_err(MlpBlockError::BackendError)?;++ let down_projection = <dyn Linear<B>>::new(+ &dense_config.linear_config,+ false,+ hidden_dimension,+ [model_dimension],+ context,+ ¶meter_tree.subtree("down_projection")?,+ ArrayId::MlpHidden,+ ArrayId::Main,+ )?;++ return Ok(Box::new(DenseMlp::new(up_projection, gate_activation, down_projection)));+ }++ if let MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {+ let mixture_of_experts_block =+ MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;+ return Ok(Box::new(mixture_of_experts_block));+ }++ unreachable!("Unknown MLP config")+ }+}diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/mlp/moe.rssimilarity index 97%rename from crates/uzu/src/encodable_block/moe_block.rsrename to crates/uzu/src/encodable_block/mlp/moe.rsindex a5be081ba..7efde157c 100644--- a/crates/uzu/src/encodable_block/moe_block.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -22,7 +22,7 @@ use crate::{},},},- encodable_block::{EncodableBlock, EncodingParameters},+ encodable_block::mlp::Mlp,forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterLoaderError, ParameterTree},};@@ -63,7 +63,7 @@ pub enum MoeBlockError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Parameter loader error: {0}")]- ParameterLoaderError(#[source] ParameterLoaderError),+ ParameterLoaderError(#[source] ParameterLoaderError<B>),}impl<B: Backend> MoeBlock<B> {@@ -92,8 +92,8 @@ impl<B: Backend> MoeBlock<B> {LinearConfig::FullPrecision {..} => {- let weights_arr = router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?;- let biases_arr = router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?;+ let weights_arr = router_tree.leaf_array("weights").map_err(MoeBlockError::ParameterLoaderError)?;+ let biases_arr = router_tree.leaf_array("biases").map_err(MoeBlockError::ParameterLoaderError)?;RouterBlock {weights_buf: weights_arr.buffer(),biases_buf: biases_arr.buffer(),@@ -128,25 +128,25 @@ impl<B: Backend> MoeBlock<B> {let w13_arr = experts_tree.subtree("up_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf("weights")+ .leaf_array("weights").map_err(MoeBlockError::ParameterLoaderError)?;let w2_arr = experts_tree.subtree("down_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf("weights")+ .leaf_array("weights").map_err(MoeBlockError::ParameterLoaderError)?;let up_biases_arr = experts_tree.subtree("up_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf("biases")+ .leaf_array("biases").map_err(MoeBlockError::ParameterLoaderError)?;let down_biases_arr = experts_tree.subtree("down_projection").map_err(MoeBlockError::ParameterLoaderError)?- .leaf("biases")+ .leaf_array("biases").map_err(MoeBlockError::ParameterLoaderError)?;let shared_weights = SharedMoeWeights {@@ -188,11 +188,10 @@ impl<B: Backend> MoeBlock<B> {}}-impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {+impl<B: Backend> Mlp<B> for MoeBlock<B> {fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let suffix_length = state.active_suffix_length();diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex a973d83c3..b29cb69d1 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -1,61 +1,44 @@-use crate::{- backends::common::{Backend, CommandBuffer},- forward_pass::state::ForwardPassState,-};-mod activation;mod attention;mod classifier_layer;mod decoder;-mod full_precision_embedding_lookup;-mod full_precision_embedding_readout;-mod full_precision_linear;+mod embedding;mod layer;mod layer_norm;+pub(crate) mod linear;mod mamba_mixer;-mod mlp;-mod moe_block;+pub(crate) mod mlp;mod normalization;mod pooling;mod prediction_head;mod qk_norm;-mod quantized_embedding_lookup;-mod quantized_embedding_readout;-mod quantized_linear;mod rms_norm;mod rope;mod sampling;mod short_conv_mixer;mod tensor_add_swap;mod tensor_copy;-mod transformer_layer;pub use activation::Activation;pub use attention::Attention;pub use classifier_layer::ClassifierLayer;pub use decoder::Decoder;-pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};-pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};-pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};+pub use embedding::Embedding;pub use layer::LayerExecutables;pub use layer_norm::{LayerNorm, LayerNormError};+pub use linear::{FullPrecisionLinear, FullPrecisionLinearError, Linear, QuantizedLinear, QuantizedLinearError};pub(crate) use mamba_mixer::MambaMixer;-pub use mlp::MlpBlock;-pub use moe_block::MoeBlock;+pub use mlp::{DenseMlp, Mlp, MoeBlock, MoeBlockError};pub use normalization::{Normalization, NormalizationError};pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;pub use qk_norm::{QKNorm, QKNormError};-pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};-pub use quantized_embedding_readout::{QuantizedEmbeddingReadout, QuantizedEmbeddingReadoutError};-pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};pub use rms_norm::{RMSNorm, RMSNormError};pub use rope::Rope;pub use sampling::Sampling;pub use short_conv_mixer::ShortConvMixer;pub use tensor_add_swap::TensorAddSwap;pub use tensor_copy::TensorCopy;-pub use transformer_layer::{embed_block, linear_block};#[derive(Clone)]pub struct EncodingParameters {@@ -77,12 +60,3 @@ impl EncodingParameters {self}}--pub trait EncodableBlock<B: Backend> {- fn encode(- &self,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error>;-}diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rsindex 308b3fab6..53d5cfd51 100644--- a/crates/uzu/src/encodable_block/normalization.rs+++ b/crates/uzu/src/encodable_block/normalization.rs@@ -2,7 +2,7 @@use thiserror::Error;-use super::{EncodableBlock, EncodingParameters, LayerNorm, LayerNormError, RMSNorm, RMSNormError};+use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};use crate::{DataType,backends::common::{Backend, CommandBuffer},@@ -56,18 +56,15 @@ impl<B: Backend> Normalization<B> {)?))}}-}-impl<B: Backend> EncodableBlock<B> for Normalization<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {match self {- Self::LayerNorm(layer_norm) => layer_norm.encode(state, parameters, command_buffer),- Self::RMSNorm(rms_norm) => rms_norm.encode(state, parameters, command_buffer),+ Self::LayerNorm(layer_norm) => layer_norm.encode(state, command_buffer),+ Self::RMSNorm(rms_norm) => rms_norm.encode(state, command_buffer),}}}diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rsindex bf8010fac..b6cfba724 100644--- a/crates/uzu/src/encodable_block/pooling.rs+++ b/crates/uzu/src/encodable_block/pooling.rs@@ -2,7 +2,6 @@use std::ops::{Deref, DerefMut};-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -58,13 +57,10 @@ impl<B: Backend> Pooling<B> {model_dim,})}-}-impl<B: Backend> EncodableBlock<B> for Pooling<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let batch_size = 1;diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rsindex 46ec73491..62e62ba44 100644--- a/crates/uzu/src/encodable_block/prediction_head.rs+++ b/crates/uzu/src/encodable_block/prediction_head.rs@@ -4,25 +4,25 @@use crate::forward_pass::state::ArrayId;use crate::{backends::common::{Backend, CommandBuffer},- encodable_block::{EncodableBlock, EncodingParameters},+ encodable_block::{Activation, Normalization, linear::Linear},forward_pass::state::ForwardPassState,};pub struct ClassifierPredictionHead<B: Backend> {- dense: Box<dyn EncodableBlock<B>>,- activation: Box<dyn EncodableBlock<B>>,- norm: Box<dyn EncodableBlock<B>>,- readout: Box<dyn EncodableBlock<B>>,+ dense: Box<dyn Linear<B>>,+ activation: Activation<B>,+ norm: Normalization<B>,+ readout: Box<dyn Linear<B>>,#[allow(dead_code)]num_labels: usize,}impl<B: Backend> ClassifierPredictionHead<B> {pub fn new(- dense: Box<dyn EncodableBlock<B>>,- activation: Box<dyn EncodableBlock<B>>,- norm: Box<dyn EncodableBlock<B>>,- readout: Box<dyn EncodableBlock<B>>,+ dense: Box<dyn Linear<B>>,+ activation: Activation<B>,+ norm: Normalization<B>,+ readout: Box<dyn Linear<B>>,num_labels: usize,) -> Self {Self {@@ -33,19 +33,16 @@ impl<B: Backend> ClassifierPredictionHead<B> {num_labels,}}-}-impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {- self.dense.encode(state, parameters, command_buffer)?;- self.activation.encode(state, parameters, command_buffer)?;- self.norm.encode(state, parameters, command_buffer)?;- self.readout.encode(state, parameters, command_buffer)?;+ self.dense.encode(state, command_buffer)?;+ self.activation.encode(state, command_buffer)?;+ self.norm.encode(state, command_buffer)?;+ self.readout.encode(state, command_buffer)?;#[cfg(feature = "tracing")]{diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rsindex 0c25f43bf..6c5a7cdab 100644--- a/crates/uzu/src/encodable_block/qk_norm.rs+++ b/crates/uzu/src/encodable_block/qk_norm.rs@@ -8,7 +8,6 @@ use std::{use thiserror::Error;-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -25,7 +24,7 @@ pub enum QKNormError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),+ ParameterError(ParameterLoaderError<B>),}pub struct QKNorm<B: Backend> {@@ -60,7 +59,7 @@ impl<B: Backend> QKNorm<B> {// Setup query normalization if configuredif let Some(ref q_config) = query_config {- let scales = parameter_tree.leaf("query_norm.scales").map_err(QKNormError::ParameterError)?;+ let scales = parameter_tree.leaf_array("query_norm.scales").map_err(QKNormError::ParameterError)?;let accumulation_data_type: DataType = q_config.accumulation_precision.into();let scale_data_type: DataType = q_config.scale_precision.into();@@ -85,7 +84,7 @@ impl<B: Backend> QKNorm<B> {// Setup key normalization if configuredif let Some(ref k_config) = key_config {- let scales = parameter_tree.leaf("key_norm.scales").map_err(QKNormError::ParameterError)?;+ let scales = parameter_tree.leaf_array("key_norm.scales").map_err(QKNormError::ParameterError)?;let accumulation_data_type: DataType = k_config.accumulation_precision.into();let scale_data_type: DataType = k_config.scale_precision.into();@@ -121,13 +120,10 @@ impl<B: Backend> QKNorm<B> {head_dim,})}-}-impl<B: Backend> EncodableBlock<B> for QKNorm<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let qkv_binding = state.arrays(&[self.qkv_array_id]);diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rsdeleted file mode 100644index ac50416da..000000000--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs+++ /dev/null@@ -1,275 +0,0 @@-//! Quantized embedding lookup encodable.--use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};--use thiserror::Error;--use super::{EncodableBlock, EncodingParameters};-use crate::{- DataType,- backends::common::{- Backend, Buffer, CommandBuffer, Context,- kernel::{Kernels, QuantizedEmbeddingLookupKernel},- },- config::QuantizationMode,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::{ParameterLoaderError, ParameterTree},-};--#[derive(Debug, Error)]-pub enum QuantizedEmbeddingLookupError<B: Backend> {- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),- #[error("Expected packed weights of type {expected:?}, got {got:?}")]- InvalidWeightsDataType {- expected: DataType,- got: DataType,- },- #[error(- "Embedding lookup weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_packed_dim}]"- )]- InvalidWeightsShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_packed_dim: usize,- },- #[error(- "Embedding lookup scales shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"- )]- InvalidScalesShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_num_groups: usize,- },- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidScalesDataType {- expected: DataType,- got: DataType,- },- #[error(- "Embedding lookup biases shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"- )]- InvalidBiasesShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_num_groups: usize,- },- #[error("Biases dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidBiasesDataType {- expected: DataType,- got: DataType,- },-}--pub struct QuantizedEmbeddingLookup<B: Backend> {- kernel: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,- weights_buffer: Rc<RefCell<B::Buffer>>,- scales_buffer: Rc<RefCell<B::Buffer>>,- biases_buffer: Rc<RefCell<B::Buffer>>,- mode: QuantizationMode,- input_scale: f32,- vocab_size: u32,- model_dim: u32,- group_size: u32,-}--impl<B: Backend> QuantizedEmbeddingLookup<B> {- pub fn new_tied(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- input_scale: f32,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, QuantizedEmbeddingLookupError<B>> {- Self::new_with_names(- context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- input_scale,- "weights",- "scales",- "biases",- parameter_tree,- )- }-- pub fn new_untied_input(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- input_scale: f32,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, QuantizedEmbeddingLookupError<B>> {- Self::new_with_names(- context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- input_scale,- "input_weights",- "input_scales",- "input_biases",- parameter_tree,- )- }-- fn new_with_names(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- input_scale: f32,- weights_name: &str,- scales_name: &str,- biases_name: &str,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, QuantizedEmbeddingLookupError<B>> {- let packing_divisor = mode.packing_divisor();-- let kernel = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(context, data_type)- .map_err(QuantizedEmbeddingLookupError::BackendError)?;-- let weights = parameter_tree.leaf(weights_name).map_err(QuantizedEmbeddingLookupError::ParameterError)?;-- if weights.data_type() != mode.storage_type() {- return Err(QuantizedEmbeddingLookupError::InvalidWeightsDataType {- expected: mode.storage_type(),- got: weights.data_type(),- });- }-- let scales = parameter_tree.leaf(scales_name).map_err(QuantizedEmbeddingLookupError::ParameterError)?;-- let num_groups = (model_dim + group_size - 1) / group_size;- if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingLookupError::InvalidWeightsShape {- got: weights.shape().to_vec().into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_packed_dim: model_dim / packing_divisor,- });- }-- if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingLookupError::InvalidScalesShape {- got: scales.shape().to_vec().into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_num_groups: num_groups,- });- }-- if scales.data_type() != data_type {- return Err(QuantizedEmbeddingLookupError::InvalidScalesDataType {- expected: data_type,- got: scales.data_type(),- });- }-- let biases_buffer = match parameter_tree.leaf(biases_name) {- Ok(biases) => {- if biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingLookupError::InvalidBiasesShape {- got: biases.shape().to_vec().into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_num_groups: num_groups,- });- }-- if biases.data_type() != data_type {- return Err(QuantizedEmbeddingLookupError::InvalidBiasesDataType {- expected: data_type,- got: biases.data_type(),- });- }-- biases.buffer()- },- Err(_) => {- let element_size = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(QuantizedEmbeddingLookupError::UnsupportedDataType(other));- },- };-- let size_bytes = vocab_size * num_groups * element_size;- let buffer = context.create_buffer(size_bytes).map_err(QuantizedEmbeddingLookupError::BackendError)?;-- unsafe {- std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);- }-- Rc::new(RefCell::new(buffer))- },- };-- Ok(Self {- kernel,- weights_buffer: weights.buffer(),- scales_buffer: scales.buffer(),- biases_buffer,- mode,- input_scale,- vocab_size: vocab_size as u32,- model_dim: model_dim as u32,- group_size: group_size as u32,- })- }-}--impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {- fn encode(- &self,- state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);- let batch_size = state.active_suffix_length();- let token_ids_array = arrays[0].borrow_mut();- let output_array = arrays[1].borrow_mut();-- let quant_mode = match self.mode {- QuantizationMode::UInt4 => 0,- QuantizationMode::Int8 => 1,- QuantizationMode::UInt8 => 2,- };-- self.kernel.encode(- token_ids_array.buffer().borrow().deref(),- self.weights_buffer.borrow().deref(),- self.scales_buffer.borrow().deref(),- self.biases_buffer.borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),- batch_size as u32,- self.vocab_size,- self.model_dim,- self.group_size,- self.input_scale,- quant_mode,- command_buffer,- );- Ok(())- }-}diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rsdeleted file mode 100644index 791237300..000000000--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs+++ /dev/null@@ -1,270 +0,0 @@-//! Quantized embedding readout encodable.--use std::{- cell::RefCell,- ops::{Deref, DerefMut},- rc::Rc,-};--use thiserror::Error;--use super::{EncodableBlock, EncodingParameters};-use crate::{- DataType,- backends::common::{- Backend, Buffer, CommandBuffer, Context,- kernel::quant_matmul::{- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,- QuantizedMatmulKernelEncodable, QuantizedMatmulType,- },- },- config::QuantizationMode,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::{ParameterLoaderError, ParameterTree},-};--#[derive(Debug, Error)]-pub enum QuantizedEmbeddingReadoutError<B: Backend> {- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("Matmul error: {0}")]- MatmulError(#[source] QuantizedMatmulError<B>),- #[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),- #[error(- "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_packed_dim}]"- )]- InvalidWeightsShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_packed_dim: usize,- },- #[error(- "Embedding readout scales shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"- )]- InvalidScalesShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_num_groups: usize,- },- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidScalesDataType {- expected: DataType,- got: DataType,- },- #[error(- "Embedding readout deq_biases shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"- )]- InvalidBiasesShape {- got: Box<[usize]>,- expected_vocab_size: usize,- expected_num_groups: usize,- },- #[error("Biases dtype mismatch: got {got:?}, expected {expected:?}")]- InvalidBiasesDataType {- expected: DataType,- got: DataType,- },-}--pub struct QuantizedEmbeddingReadout<B: Backend> {- kernel: QuantizedMatmulKernelEncodable<B>,- weights_buffer: Rc<RefCell<B::Buffer>>,- scales_buffer: Rc<RefCell<B::Buffer>>,- biases_buffer: Rc<RefCell<B::Buffer>>,- vocab_size: usize,- model_dim: usize,-}--impl<B: Backend> QuantizedEmbeddingReadout<B> {- pub fn new_tied(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {- Self::new_with_names(- context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- "weights",- "scales",- "biases",- parameter_tree,- )- }-- pub fn new_untied_output(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {- Self::new_with_names(- context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- "output_weights",- "output_scales",- "output_biases",- parameter_tree,- )- }-- fn new_with_names(- context: &B::Context,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- weights_name: &str,- scales_name: &str,- biases_name: &str,- parameter_tree: &ParameterTree<B::Context>,- ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {- let weights = parameter_tree.leaf(weights_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;- let scales = parameter_tree.leaf(scales_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;-- let num_groups = (model_dim + group_size - 1) / group_size;- let packing_divisor = mode.packing_divisor();- let weights_transposed = weights.shape()[0] == vocab_size;-- if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingReadoutError::InvalidWeightsShape {- got: weights.shape().to_vec().into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_packed_dim: model_dim / packing_divisor,- });- }-- if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingReadoutError::InvalidScalesShape {- got: scales.shape().to_vec().into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_num_groups: num_groups,- });- }-- if scales.data_type() != data_type {- return Err(QuantizedEmbeddingReadoutError::InvalidScalesDataType {- expected: data_type,- got: scales.data_type(),- });- }-- let biases_buffer = match parameter_tree.leaf(biases_name) {- Ok(deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingReadoutError::InvalidBiasesShape {- got: deq_biases.shape().to_vec().into_boxed_slice(),- expected_vocab_size: vocab_size,- expected_num_groups: num_groups,- });- }- if deq_biases.data_type() != data_type {- return Err(QuantizedEmbeddingReadoutError::InvalidBiasesDataType {- expected: data_type,- got: deq_biases.data_type(),- });- }- deq_biases.buffer()- },- Err(_) => {- let element_size = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(QuantizedEmbeddingReadoutError::UnsupportedDataType(other));- },- };- let size_bytes = vocab_size * num_groups * element_size;- let buffer = context.create_buffer(size_bytes).map_err(QuantizedEmbeddingReadoutError::BackendError)?;-- unsafe {- std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);- }-- Rc::new(RefCell::new(buffer))- },- };-- let kernel = QuantizedMatmulKernelEncodable::new(- context,- QuantizedMatmulConfiguration {- data_type,- group_size,- input_dim: model_dim,- output_dim: vocab_size,- mode,- quantization_type: QuantizedMatmulType::Mlx,- weights_transposed,- },- )- .map_err(QuantizedEmbeddingReadoutError::MatmulError)?;-- Ok(Self {- kernel,- weights_buffer: weights.buffer(),- scales_buffer: scales.buffer(),- biases_buffer,- vocab_size,- model_dim,- })- }-}--impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {- fn encode(- &self,- state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,- ) -> Result<(), B::Error> {- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);- let batch_size = state.sampling_length();- if batch_size == 0 {- return Ok(());- }-- let sampling_start = state.sampling_start();- let input_array = arrays[0].borrow_mut();- let output_array = arrays[1].borrow_mut();-- let element_size = input_array.data_type().size_in_bytes();- let a_offset = sampling_start * self.model_dim * element_size;-- self.kernel- .encode(- command_buffer,- QuantizedMatmulArguments {- a_buffer: input_array.buffer().borrow().deref(),- a_offset,- b_buffer: self.weights_buffer.borrow().deref(),- scales_buffer: self.scales_buffer.borrow().deref(),- zero_points_or_biases_buffer: self.biases_buffer.borrow().deref(),- output_buffer: output_array.buffer().borrow_mut().deref_mut(),- batch: batch_size,- input_dim: self.model_dim,- output_dim: self.vocab_size,- quantization_type: QuantizedMatmulType::Mlx,- },- )- .expect("Failed to encode quantized embedding readout kernel");- Ok(())- }-}diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rsindex 13a1c7483..30ecb88f1 100644--- a/crates/uzu/src/encodable_block/rms_norm.rs+++ b/crates/uzu/src/encodable_block/rms_norm.rs@@ -8,7 +8,6 @@ use std::{use thiserror::Error;-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -25,7 +24,7 @@ pub enum RMSNormError<B: Backend> {#[error("Backend error: {0}")]BackendError(#[source] B::Error),#[error("Parameter loading error: {0}")]- ParameterError(ParameterLoaderError),+ ParameterError(ParameterLoaderError<B>),}pub struct RMSNorm<B: Backend> {@@ -46,7 +45,7 @@ impl<B: Backend> RMSNorm<B> {output_array_id: ArrayId,parameter_tree: &ParameterTree<B::Context>,) -> Result<Self, RMSNormError<B>> {- let scales = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;+ let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;let accumulation_data_type: DataType = config.accumulation_precision.into();let scale_data_type: DataType = config.scale_precision.into();@@ -82,13 +81,10 @@ impl<B: Backend> RMSNorm<B> {self.use_sampling_range = true;self}-}-impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let input_binding = state.arrays(&[self.input_array_id]);diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rsindex 66ca919f2..4e3aec68c 100644--- a/crates/uzu/src/encodable_block/rope.rs+++ b/crates/uzu/src/encodable_block/rope.rs@@ -2,7 +2,6 @@use std::ops::{Deref, DerefMut};-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -28,13 +27,10 @@ impl<B: Backend> Rope<B> {rope_type,})}-}-impl<B: Backend> EncodableBlock<B> for Rope<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rsindex 2b4901f0b..f63ff47a5 100644--- a/crates/uzu/src/encodable_block/sampling.rs+++ b/crates/uzu/src/encodable_block/sampling.rs@@ -2,7 +2,6 @@use std::ops::{Deref, DerefMut};-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -42,13 +41,10 @@ impl<B: Backend> Sampling<B> {kernel,})}-}-impl<B: Backend> EncodableBlock<B> for Sampling<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex 75534176f..98406a2e7 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -8,7 +8,7 @@ use crate::{kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},},config::{DecoderLayerType, ShortConvConfig},- encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},+ encodable_block::linear::Linear,forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterTree, resolve_subtree},};@@ -17,8 +17,8 @@ pub struct ShortConvMixer<B: Backend> {layer_index: usize,config: ShortConvConfig,model_dim: usize,- in_projection: Box<dyn EncodableBlock<B>>,- out_projection: Box<dyn EncodableBlock<B>>,+ in_projection: Box<dyn Linear<B>>,+ out_projection: Box<dyn Linear<B>>,short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,short_conv_prefill: <B::Kernels as Kernels>::ShortConvPrefillKernel,short_conv_decode: <B::Kernels as Kernels>::ShortConvDecodeKernel,@@ -45,7 +45,7 @@ impl<B: Backend> ShortConvMixer<B> {let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();- let in_projection = linear_block(+ let in_projection = <dyn Linear<B>>::new(&short_conv_config.in_projection_config,false,model_dim,@@ -57,7 +57,7 @@ impl<B: Backend> ShortConvMixer<B> {).expect("Failed to create in-projection kernel");- let out_projection = linear_block(+ let out_projection = <dyn Linear<B>>::new(&short_conv_config.out_projection_config,false,model_dim,@@ -69,9 +69,9 @@ impl<B: Backend> ShortConvMixer<B> {).expect("Failed to create out-projection kernel");- let conv_weight = conv_tree.leaf("weights").unwrap().clone();+ let conv_weight = conv_tree.leaf_array("weights").unwrap().clone();let conv_bias = if short_conv_config.conv_config.has_biases {- Some(conv_tree.leaf("biases").unwrap().clone())+ Some(conv_tree.leaf_array("biases").unwrap().clone())} else {None};@@ -100,9 +100,7 @@ impl<B: Backend> ShortConvMixer<B> {conv_bias,}}-}-impl<B: Backend> ShortConvMixer<B> {fn clear_suffix_state_valid_range(&self,state: &ForwardPassState<B>,@@ -322,13 +320,10 @@ impl<B: Backend> ShortConvMixer<B> {compute,)}-}-impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let active_suffix_length = state.active_suffix_length();@@ -336,11 +331,11 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {return Ok(());}- self.in_projection.encode(state, parameters, command_buffer)?;+ self.in_projection.encode(state, command_buffer)?;self.run_conv(state, command_buffer, active_suffix_length);- self.out_projection.encode(state, parameters, command_buffer)?;+ self.out_projection.encode(state, command_buffer)?;Ok(())}diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rsindex 627208b12..ea0b2b9d1 100644--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs@@ -2,7 +2,6 @@use std::ops::DerefMut;-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -29,13 +28,10 @@ impl<B: Backend> TensorAddSwap<B> {argument_arrays,})}-}-impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let arrays = state.arrays(&self.argument_arrays);diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rsindex d82361ead..74af8ac92 100644--- a/crates/uzu/src/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/encodable_block/tensor_copy.rs@@ -2,7 +2,6 @@use std::ops::{Deref, DerefMut};-use super::{EncodableBlock, EncodingParameters};use crate::{DataType,backends::common::{@@ -29,13 +28,10 @@ impl<B: Backend> TensorCopy<B> {argument_arrays,})}-}-impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {- fn encode(+ pub fn encode(&self,state: &mut ForwardPassState<B>,- _parameters: &EncodingParameters,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<(), B::Error> {let arrays = state.arrays(&self.argument_arrays);diff --git a/crates/uzu/src/encodable_block/transformer_layer.rs b/crates/uzu/src/encodable_block/transformer_layer.rsdeleted file mode 100644index 736fe8fb3..000000000--- a/crates/uzu/src/encodable_block/transformer_layer.rs+++ /dev/null@@ -1,405 +0,0 @@-use thiserror::Error;--use super::{- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, QuantizedEmbeddingLookup,- QuantizedEmbeddingReadout, QuantizedLinear,-};-use crate::{- DataType,- backends::common::{Backend, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},- config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},- encodable_block::{- EncodableBlock, FullPrecisionLinearError, MlpBlock, MoeBlock, QuantizedLinearError, moe_block::MoeBlockError,- },- forward_pass::state::ArrayId,- parameters::{ParameterLoaderError, ParameterTree},-};--#[derive(Debug, Error)]-pub enum LayerError<B: Backend> {- #[error("Backend error: {0}")]- BackendError(#[source] B::Error),- #[error("QuantizedLinear error: {0}")]- QuantizedLinearError(#[source] QuantizedLinearError<B>),- #[error("FullPrecisionLinear error: {0}")]- FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),- #[error("MoeBlock error: {0}")]- MoeBlockError(#[source] MoeBlockError<B>),- #[error("Parameter loader error: {0}")]- ParameterLoaderError(#[source] ParameterLoaderError),- #[error("QLoRA linear layer not supported")]- QLoRaNotSupported,-}--pub fn linear_block<const N: usize, B: Backend>(- config: &LinearConfig,- _has_biases: bool,- input_dimension: usize,- output_dimensions: [usize; N],- context: &B::Context,- parameter_tree: &ParameterTree<B::Context>,- input_array_id: ArrayId,- output_array_id: ArrayId,-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {- let output_dimension_sum: usize = output_dimensions.iter().sum();- match config {- LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {- let block = QuantizedLinear::new(- context,- quantization_config,- input_dimension,- output_dimension_sum,- parameter_tree,- input_array_id,- output_array_id,- )- .map_err(LayerError::QuantizedLinearError)?;- Ok(Box::new(block))- },- LinearConfig::FullPrecision {- precision,- } => {- let block = FullPrecisionLinear::new(- context,- (*precision).into(),- input_dimension,- output_dimension_sum,- parameter_tree,- input_array_id,- output_array_id,- )- .map_err(LayerError::FullPrecisionLinearError)?;- Ok(Box::new(block))- },- LinearConfig::QLoRA {- ..- } => Err(LayerError::QLoRaNotSupported),- }-}--/// Creates an MLP block using the unfused implementation (separate up, activation, down)-pub fn mlp_block<B: Backend>(- config: &MLPConfig,- model_dimension: usize,- hidden_dimension: usize,- context: &B::Context,- parameter_tree: &ParameterTree<B::Context>,-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {- if let crate::config::MLPConfig::Dense(dense_config) = config {- let data_type: DataType = dense_config.linear_config.activation_precision().into();-- // Up projection (outputs 2*hidden_dimension for gate and up)- let up_projection = linear_block(- &dense_config.linear_config,- false,- model_dimension,- [2 * hidden_dimension],- context,- ¶meter_tree.subtree("up_projection").map_err(LayerError::ParameterLoaderError)?,- ArrayId::Main,- ArrayId::MlpFusedUp,- )?;-- // Gate activation + multiply- let gate_activation =- MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)- .map_err(LayerError::BackendError)?;-- // Down projection- let down_projection = linear_block(- &dense_config.linear_config,- false,- hidden_dimension,- [model_dimension],- context,- ¶meter_tree.subtree("down_projection").map_err(LayerError::ParameterLoaderError)?,- ArrayId::MlpHidden,- ArrayId::Main,- )?;-- return Ok(Box::new(MlpBlock::new(up_projection, gate_activation, down_projection)));- }-- if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {- let mixture_of_experts_block =- MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)- .map_err(LayerError::MoeBlockError)?;- return Ok(Box::new(mixture_of_experts_block));- }-- unreachable!("Unknown MLP config")-}--pub fn embed_block<B: Backend>(- config: &DecoderConfig,- context: &B::Context,- parameter_tree: &ParameterTree<B::Context>,-) -> Box<dyn EncodableBlock<B>> {- match &config.embedding_config {- EmbeddingConfig::Tied {- common,- precision,- } => {- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = FullPrecisionEmbeddingLookup::new(- context,- (*precision).into(),- config.vocab_size,- config.model_dim,- common.input_scale,- &embeddings_tree,- )- .expect("Failed to create full precision embedding lookup");-- Box::new(block)- },- EmbeddingConfig::Untied {- common,- precision,- } => {- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = FullPrecisionEmbeddingLookup::new(- context,- (*precision).into(),- config.vocab_size,- config.model_dim,- common.input_scale,- &embeddings_tree,- )- .expect("Failed to create full precision embedding lookup");-- Box::new(block)- },- EmbeddingConfig::MLXSemiQuantizedUntied {- common,- activation_precision,- ..- } => {- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = FullPrecisionEmbeddingLookup::new(- context,- (*activation_precision).into(),- config.vocab_size,- config.model_dim,- common.input_scale,- &embeddings_tree,- )- .expect("Failed to create full precision embedding lookup");-- Box::new(block)- },- EmbeddingConfig::MLXQuantizedUntied {- group_size,- embedding_quantization_mode,- activation_precision,- ..- } => {- let data_type: DataType = (*activation_precision).into();- let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);-- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = QuantizedEmbeddingLookup::new_untied_input(- context,- data_type,- config.vocab_size,- config.model_dim,- *group_size,- *embedding_quantization_mode,- input_scale,- &embeddings_tree,- )- .expect("Failed to create quantized embedding lookup kernel");-- Box::new(block)- },- EmbeddingConfig::QuantizedTied {- embedding_quantization_mode,- activation_precision,- ..- } => {- let data_type: DataType = (*activation_precision).into();-- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim- let group_size = config.model_dim;- let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);-- let block = QuantizedEmbeddingLookup::new_tied(- context,- data_type,- config.vocab_size,- config.model_dim,- group_size,- *embedding_quantization_mode,- input_scale,- &embeddings_tree,- )- .expect("Failed to create quantized embedding lookup kernel");-- Box::new(block)- },- EmbeddingConfig::MLXQuantizedTied {- group_size,- embedding_quantization_mode,- ..- } => {- let data_type: DataType = config.output_norm_config.scale_precision.into();- let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);-- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = QuantizedEmbeddingLookup::new_tied(- context,- data_type,- config.vocab_size,- config.model_dim,- *group_size,- *embedding_quantization_mode,- input_scale,- &embeddings_tree,- )- .expect("Failed to create quantized embedding lookup kernel");-- Box::new(block)- },- }-}--pub fn readout_block<B: Backend>(- config: &DecoderConfig,- context: &B::Context,- parameter_tree: &ParameterTree<B::Context>,-) -> Box<dyn EncodableBlock<B>> {- match &config.embedding_config {- EmbeddingConfig::Tied {- precision,- ..- } => {- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = FullPrecisionEmbeddingReadout::new(- context,- (*precision).into(),- config.vocab_size,- config.model_dim,- &embeddings_tree,- )- .expect("Failed to create full precision embedding readout");-- Box::new(block)- },- EmbeddingConfig::Untied {- precision,- ..- } => {- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = FullPrecisionEmbeddingReadout::new(- context,- (*precision).into(),- config.vocab_size,- config.model_dim,- &embeddings_tree,- )- .expect("Failed to create full precision embedding readout");-- Box::new(block)- },- EmbeddingConfig::MLXSemiQuantizedUntied {- group_size,- activation_precision,- embedding_quantization_mode,- ..- } => {- let data_type: DataType = (*activation_precision).into();- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = QuantizedEmbeddingReadout::new_untied_output(- context,- data_type,- config.vocab_size,- config.model_dim,- *group_size,- *embedding_quantization_mode,- &embeddings_tree,- )- .expect("Failed to create quantized embedding readout kernel");-- Box::new(block)- },- EmbeddingConfig::QuantizedTied {- embedding_quantization_mode,- activation_precision,- ..- } => {- let data_type: DataType = (*activation_precision).into();-- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim- let group_size = config.model_dim;-- let block = QuantizedEmbeddingReadout::new_tied(- context,- data_type,- config.vocab_size,- config.model_dim,- group_size,- *embedding_quantization_mode,- &embeddings_tree,- )- .expect("Failed to create quantized embedding readout kernel");-- Box::new(block)- },- EmbeddingConfig::MLXQuantizedTied {- group_size,- embedding_quantization_mode,- ..- } => {- let data_type: DataType = config.output_norm_config.scale_precision.into();-- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = QuantizedEmbeddingReadout::new_tied(- context,- data_type,- config.vocab_size,- config.model_dim,- *group_size,- *embedding_quantization_mode,- &embeddings_tree,- )- .expect("Failed to create quantized embedding readout kernel");-- Box::new(block)- },- EmbeddingConfig::MLXQuantizedUntied {- group_size,- embedding_quantization_mode,- ..- } => {- let data_type: DataType = config.output_norm_config.scale_precision.into();-- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");-- let block = QuantizedEmbeddingReadout::new_untied_output(- context,- data_type,- config.vocab_size,- config.model_dim,- *group_size,- *embedding_quantization_mode,- &embeddings_tree,- )- .expect("Failed to create quantized embedding readout kernel");-- Box::new(block)- },- }-}diff --git a/crates/uzu/src/forward_pass/state/rope_buffers.rs b/crates/uzu/src/forward_pass/state/rope_buffers.rsindex d21a65db2..628e63e12 100644--- a/crates/uzu/src/forward_pass/state/rope_buffers.rs+++ b/crates/uzu/src/forward_pass/state/rope_buffers.rs@@ -46,10 +46,10 @@ impl<B: Backend> RopeBuffers<B> {return;};- let cosines_view = rope_tree.leaf("cosines").unwrap();+ let cosines_view = rope_tree.leaf_array("cosines").unwrap();self.cosines.borrow_mut().copy_from_array(&cosines_view);- let sines_view = rope_tree.leaf("sines").unwrap();+ let sines_view = rope_tree.leaf_array("sines").unwrap();self.sines.borrow_mut().copy_from_array(&sines_view);}}diff --git a/crates/uzu/src/forward_pass/state/shared_buffers.rs b/crates/uzu/src/forward_pass/state/shared_buffers.rsindex 2d2e02980..2cfa6b6e6 100644--- a/crates/uzu/src/forward_pass/state/shared_buffers.rs+++ b/crates/uzu/src/forward_pass/state/shared_buffers.rs@@ -65,7 +65,7 @@ impl<B: Backend> SharedBuffers<B> {for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();let attn_tree = layer_tree.subtree("mixer").unwrap();- let sinks_arr = attn_tree.leaf("sinks").unwrap();+ let sinks_arr = attn_tree.leaf_array("sinks").unwrap();let mut dst = sink_cell.borrow_mut();let dst_slice = dst.as_slice_mut::<f32>();diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex dcc57048d..7202de709 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -21,7 +21,7 @@ use crate::{CommandBufferPending, Context,kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},},- encodable_block::{EncodableBlock, EncodingParameters},+ encodable_block::EncodingParameters,forward_pass::{cache_layers::{CacheLayer, CacheLayersSlice},kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},@@ -536,7 +536,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// Encode samplingself.context.gpu_sampler- .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)+ .encode(&mut state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;// Copy sampled token: sampling_output → token_ids (for next pass)@@ -885,7 +885,7 @@ impl<B: Backend> LanguageModelGenerator<B> {if sample {self.context.gpu_sampler- .encode(state, parameters, &mut command_buffer)+ .encode(state, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;}diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rsindex be1380602..529a8361e 100644--- a/crates/uzu/src/parameters/loader.rs+++ b/crates/uzu/src/parameters/loader.rs@@ -10,7 +10,7 @@ use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingErroruse crate::{DataType,array::{Array, ArrayContextExt},- backends::common::Context,+ backends::common::{Backend, Buffer, Context},};pub struct ParameterMetadata {@@ -43,7 +43,7 @@ fn st_metadata_into_index(}#[derive(Debug, Error)]-pub enum ParameterLoaderError {+pub enum ParameterLoaderError<B: Backend> {#[error("Array with key \"{0}\" not found.")]KeyNotFound(String),#[error("Couldn't find any arrays with prefix \"{0}\".")]@@ -58,6 +58,8 @@ pub enum ParameterLoaderError {expected_size: usize,actual_size: usize,},+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),#[error("Failed to read data")]ArrayLoadingError(#[from] std::io::Error),}@@ -92,10 +94,21 @@ whereself.index.keys()}+ pub fn get_leaf<'leaf>(+ &'leaf self,+ key: &str,+ ) -> Result<ParameterLeaf<'file, 'context, 'leaf, C>, ParameterLoaderError<C::Backend>> {+ Ok(ParameterLeaf {+ key: key.to_string(),+ metadata: self.index.get(key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.to_string()))?,+ loader: self,+ })+ }+pub fn get(&self,key: &str,- ) -> Result<Array<C::Backend>, ParameterLoaderError> {+ ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;let (offset, size) = (metadata_entry.offset, metadata_entry.size);let array_key = key.replace(".", "_");@@ -120,7 +133,7 @@ wherebuf: &mut [u8],shape: &mut Box<[usize]>,data_type: &mut DataType,- ) -> Result<(), ParameterLoaderError> {+ ) -> Result<(), ParameterLoaderError<C::Backend>> {let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;self.file.read_exact_at(buf, metadata_entry.offset as u64)?;*shape = metadata_entry.shape.to_owned();@@ -136,6 +149,37 @@ where}}+pub struct ParameterLeaf<'file, 'context, 'leaf, C: Context> {+ key: String,+ metadata: &'leaf ParameterMetadata,+ loader: &'leaf ParameterLoader<'context, 'file, C>,+}++impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C> {+ pub fn shape(&self) -> &[usize] {+ &self.metadata.shape+ }++ pub fn data_type(&self) -> DataType {+ self.metadata.data_type+ }++ pub fn size(&self) -> usize {+ self.metadata.size+ }++ pub fn read_buffer(&self) -> Result<<C::Backend as Backend>::Buffer, ParameterLoaderError<C::Backend>> {+ let mut buffer =+ self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;+ buffer.set_label(Some(&format!("parameter_loader_{}", self.key.replace(".", "_"))));+ self.loader.file.read_exact_at(+ unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, self.metadata.size) },+ self.metadata.offset as u64,+ )?;+ Ok(buffer)+ }+}+pub struct ParameterTree<'loader, C: Context> {loader: &'loader ParameterLoader<'loader, 'loader, C>,prefix: Option<String>,@@ -163,7 +207,7 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {pub fn subtree(&self,name: &str,- ) -> Result<Self, ParameterLoaderError> {+ ) -> Result<Self, ParameterLoaderError<C::Backend>> {let new_prefix = self.join_prefix(name);let num_suffixes = self.loader.keys().filter_map(|suffix| suffix.strip_prefix(&new_prefix)).count();if num_suffixes > 0 {@@ -176,20 +220,27 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {}}- pub fn leaf(+ pub fn leaf_array(&self,name: &str,- ) -> Result<Array<C::Backend>, ParameterLoaderError> {+ ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {self.loader.get(&self.join_prefix(name))}+ pub fn leaf<'leaf>(+ &'leaf self,+ name: &str,+ ) -> Result<ParameterLeaf<'loader, 'loader, 'leaf, C>, ParameterLoaderError<C::Backend>> {+ self.loader.get_leaf(&self.join_prefix(name))+ }+pub fn read_extract_at(&self,name: &str,buf: &mut [u8],shape: &mut Box<[usize]>,data_type: &mut DataType,- ) -> Result<(), ParameterLoaderError> {+ ) -> Result<(), ParameterLoaderError<C::Backend>> {self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)}}diff --git a/crates/uzu/src/parameters/mod.rs b/crates/uzu/src/parameters/mod.rsindex 903847308..c5ebfd4ac 100644--- a/crates/uzu/src/parameters/mod.rs+++ b/crates/uzu/src/parameters/mod.rs@@ -1,5 +1,5 @@mod loader;-pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};+pub use loader::{ParameterLeaf, ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};mod safetensors_metadata;// Re-export the safetensors header reader so other modules (e.g. decoderdiff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex 027b25f73..848bcb579 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -24,7 +24,7 @@ use crate::{},classifier::Classifier,config::ModelMetadata,- encodable_block::{EncodableBlock, EncodingParameters, Sampling},+ encodable_block::{EncodingParameters, Sampling},forward_pass::{cache_layers::CacheLayers,scratch_buffers::ScratchBuffers,@@ -297,7 +297,7 @@ impl<B: Backend> TraceValidator<B> {for index in transformer_layers {let arrays = state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);- if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.keys", index)) {+ if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {let keys = arrays[0].borrow();results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.keys", index),@@ -305,7 +305,7 @@ impl<B: Backend> TraceValidator<B> {});}- if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.values", index)) {+ if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {let values = arrays[1].borrow();results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.values", index),@@ -329,7 +329,7 @@ impl<B: Backend> TraceValidator<B> {format!("updated_state.{}.conv_state", index),format!("activation_trace.layer_results.{}.updated_state.conv_state", index),] {- if let Ok(expected) = traces_view.leaf(&path) {+ if let Ok(expected) = traces_view.leaf_array(&path) {results.push(TracerValidationResult {name: path,metrics: Self::validate_array(@@ -346,7 +346,7 @@ impl<B: Backend> TraceValidator<B> {format!("updated_state.{}.ssm_state", index),format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),] {- if let Ok(expected) = traces_view.leaf(&path) {+ if let Ok(expected) = traces_view.leaf_array(&path) {results.push(TracerValidationResult {name: path,metrics: Self::validate_array(data_type, &expected, &ssm_state, None),@@ -356,7 +356,7 @@ impl<B: Backend> TraceValidator<B> {}// LLM-specific: Token comparison- let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf("logits") {+ let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {let expected_tokens = Self::get_tokens_from_logits(&expected_logits);let produced_tokens = Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());expected_tokens@@ -396,8 +396,8 @@ impl<B: Backend> TraceValidator<B> {ParameterLoader::new(&traces_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;let traces_view = traces_loader.tree();- let has_token_ids = traces_view.leaf("activation_trace.token_ids").is_ok();- let has_token_positions = traces_view.leaf("activation_trace.token_positions").is_ok();+ let has_token_ids = traces_view.leaf_array("activation_trace.token_ids").is_ok();+ let has_token_positions = traces_view.leaf_array("activation_trace.token_positions").is_ok();if !has_token_ids || !has_token_positions {return Ok(Self::handle_missing_tokens(&traces_view));@@ -428,7 +428,7 @@ impl<B: Backend> TraceValidator<B> {}fn handle_missing_tokens(traces_view: &ParameterTree<B::Context>) -> TracerValidationResults {- if let Ok(expected_logits) = traces_view.leaf("logits") {+ if let Ok(expected_logits) = traces_view.leaf_array("logits") {let reference_shape = expected_logits.shape().to_vec();let metrics = TracerValidationMetrics {atol: 0.0,@@ -479,7 +479,7 @@ impl<B: Backend> TraceValidator<B> {let mut results = Vec::new();let validate = |path: &str, array: &Ref<Array<B>>| -> Option<TracerValidationResult> {- if traces_view.leaf(path).is_ok() {+ if traces_view.leaf_array(path).is_ok() {Some(TracerValidationResult {name: path.to_string(),metrics: Self::validate_array_with_name(data_type, traces_view, path, array),@@ -549,7 +549,7 @@ impl<B: Backend> TraceValidator<B> {// Embedding norm (classifier-specific)if let Some(embedding_norm) = &traces.borrow().embedding_norm {- if traces_view.leaf("activation_trace.embedding_norm").is_ok() {+ if traces_view.leaf_array("activation_trace.embedding_norm").is_ok() {results.push(TracerValidationResult {name: "activation_trace.embedding_norm".to_string(),metrics: Self::validate_array_with_name(@@ -564,7 +564,7 @@ impl<B: Backend> TraceValidator<B> {// Output pooling (classifier-specific)if let Some(output_pooling) = &traces.borrow().output_pooling {- if traces_view.leaf("activation_trace.output_pooling").is_ok() {+ if traces_view.leaf_array("activation_trace.output_pooling").is_ok() {results.push(TracerValidationResult {name: "activation_trace.output_pooling".to_string(),metrics: Self::validate_array_with_name(@@ -604,7 +604,7 @@ impl<B: Backend> TraceValidator<B> {expected_array_path: &str,produced_array: &Ref<Array<B>>,) -> TracerValidationMetrics {- let expected_array = traces_view.leaf(expected_array_path).unwrap();+ let expected_array = traces_view.leaf_array(expected_array_path).unwrap();Self::validate_array(data_type, &expected_array, produced_array, None)}@@ -783,7 +783,7 @@ impl<B: Backend> TraceValidator<B> {traces_view: &ParameterTree<B::Context>,name: &str,) -> Vec<TargetPrecision> {- let array = traces_view.leaf(name).unwrap();+ let array = traces_view.leaf_array(name).unwrap();let slice = array.as_slice::<SourcePrecision>();slice.iter().map(|x| NumCast::from(*x).unwrap()).collect()}diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rsindex c32a7990c..f9ef609cb 100644--- a/crates/uzu/tests/parameters_loader_test.rs+++ b/crates/uzu/tests/parameters_loader_test.rs@@ -24,6 +24,6 @@ fn test_parameter_loader_basic() {// tree API checklet subtree = loader.tree().subtree("embedding").unwrap();- let same = subtree.leaf("weights").unwrap();+ let same = subtree.leaf_array("weights").unwrap();assert_eq!(view, same.as_view::<bf16>());}