Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Remove EncodableBlock trait (#247)

uuuvncc9034396820
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index f19235562..7fa0e433a 100644
--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -15,9 +15,13 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[allow(unused)] batch_size: u32,
#[allow(unused)] vocab_size: u32,
#[allow(unused)] model_dim: u32,
- #[allow(unused)] group_size: u32,
#[allow(unused)] input_scale: f32,
- #[allow(unused)] quant_mode: u32,
+ #[allow(unused)]
+ #[specialize]
+ group_size: u32,
+ #[allow(unused)]
+ #[specialize]
+ quant_mode: u32,
) {
todo!()
}
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 9e0a9225e..74cfc6113 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -18,9 +18,9 @@ KERNEL(QuantizedEmbeddingLookup) (
constant uint32_t& batch_size,
constant uint32_t& vocab_size,
constant uint32_t& model_dim,
- constant uint32_t& group_size,
constant float& input_scale,
- const constant uint& quant_mode,
+ const uint32_t group_size SPECIALIZE,
+ const uint quant_mode SPECIALIZE,
const uint dim_idx AXIS(model_dim, 16),
const uint batch_idx AXIS(batch_size, 16)
) {
@@ -62,4 +62,4 @@ KERNEL(QuantizedEmbeddingLookup) (
float out_f = float(scale) * float(quantized_value) + float(bias);
out_f *= input_scale;
output[thread_position_in_grid] = T(out_f);
-}
\ No newline at end of file
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
deleted file mode 100644
index 889e3df5e..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-// Shader files only — Rust logic has moved to common/kernel/matmul/gemv/
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
deleted file mode 100644
index b4607a3dd..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-// Shader files only — Rust logic has moved to common/kernel/matmul/split_k/
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index fc2295746..0dd5bf55a 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -12,7 +12,7 @@ use crate::{
backends::common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
},
- encodable_block::{EncodableBlock, EncodingParameters},
+ encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
};
@@ -124,11 +124,11 @@ impl<B: Backend> Classifier<B> {
self.context
.embed
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode_lookup(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.embedding_norm
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
{
@@ -143,7 +143,7 @@ impl<B: Backend> Classifier<B> {
}
self.context
.output_norm
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
{
@@ -153,12 +153,12 @@ impl<B: Backend> Classifier<B> {
self.context
.pooling
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.prediction_head
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
command_buffer
@@ -204,11 +204,11 @@ impl<B: Backend> Classifier<B> {
self.context
.embed
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode_lookup(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.embedding_norm
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
for layer in self.context.layers.iter() {
layer
@@ -217,15 +217,15 @@ impl<B: Backend> Classifier<B> {
}
self.context
.output_norm
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.pooling
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.prediction_head
- .encode(&mut state, &encoding_params, &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
command_buffer
diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rs
index 3edbfd5d1..2b3a6e4e1 100644
--- a/crates/uzu/src/classifier/classifier_context.rs
+++ b/crates/uzu/src/classifier/classifier_context.rs
@@ -6,8 +6,7 @@ use crate::{
classifier::ClassifierError,
config::{ClassifierModelConfig, ModelMetadata},
encodable_block::{
- Activation, ClassifierLayer, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope,
- embed_block, linear_block,
+ Activation, ClassifierLayer, ClassifierPredictionHead, Embedding, Linear, Normalization, Pooling, Rope,
},
forward_pass::{
model_shape::ModelShape,
@@ -27,15 +26,15 @@ pub struct ClassifierContext<B: Backend> {
pub model_config: ClassifierModelConfig,
pub model_shape: ModelShape,
- pub embed: Box<dyn EncodableBlock<B>>,
+ pub embed: Embedding<B>,
pub embedding_norm: Normalization<B>,
pub layers: Box<[ClassifierLayer<B>]>,
pub output_norm: Normalization<B>,
- pub global_rope: Rc<Box<dyn EncodableBlock<B>>>,
- pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
+ pub global_rope: Rc<Rope<B>>,
+ pub local_rope: Option<Rc<Rope<B>>>,
- pub pooling: Box<dyn EncodableBlock<B>>,
- pub prediction_head: Box<dyn EncodableBlock<B>>,
+ pub pooling: Pooling<B>,
+ pub prediction_head: ClassifierPredictionHead<B>,
pub sigmoid_kernel: <B::Kernels as Kernels>::SigmoidKernel,
}
@@ -90,7 +89,14 @@ impl<B: Backend> ClassifierContext<B> {
.activation_precision()
.into();
- let embed = embed_block(&decoder_config, context.as_ref(), &root_loader_view);
+ let embed = Embedding::new(
+ context.as_ref(),
+ decoder_config.vocab_size as u32,
+ decoder_config.model_dim as u32,
+ &decoder_config.embedding_config,
+ &root_loader_view.subtree("embedding").expect("Failed to get embedding subtree"),
+ )
+ .expect("Failed to create embedding");
let global_rope = Self::create_rope_block(&context, data_type, RopeType::Global).map_err(Error::Classifier)?;
let local_rope = classifier_model_config
@@ -192,7 +198,7 @@ impl<B: Backend> ClassifierContext<B> {
let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {
Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))
})?;
- let prediction_head_dense = linear_block::<1, B>(
+ let prediction_head_dense = <dyn Linear<B>>::new::<1>(
&prediction_head_config.dense_config,
prediction_head_config.use_dense_bias,
model_dim,
@@ -203,18 +209,16 @@ impl<B: Backend> ClassifierContext<B> {
ArrayId::ClassifierPredictionHeadDense,
);
- let prediction_head_activation = Box::new(
- Activation::<B>::new(
- &context,
- prediction_head_data_type,
- prediction_head_config.activation.clone(),
- ArrayId::ClassifierPredictionHeadDense,
- ArrayId::ClassifierPredictionHeadDense,
- )
- .map_err(|e| {
- Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))
- })?,
- );
+ let prediction_head_activation = Activation::<B>::new(
+ &context,
+ prediction_head_data_type,
+ prediction_head_config.activation.clone(),
+ ArrayId::ClassifierPredictionHeadDense,
+ ArrayId::ClassifierPredictionHeadDense,
+ )
+ .map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head activation: {:?}", e)))
+ })?;
let prediction_head_norm_tree = prediction_head_tree.subtree("norm").map_err(|_| {
Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.norm".to_string()))
@@ -234,7 +238,7 @@ impl<B: Backend> ClassifierContext<B> {
let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {
Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))
})?;
- let prediction_head_final_linear = linear_block::<1, B>(
+ let prediction_head_final_linear = <dyn Linear<B>>::new::<1>(
&prediction_head_config.readout_config,
true,
model_dim,
@@ -253,28 +257,26 @@ impl<B: Backend> ClassifierContext<B> {
Error::UnableToCreateContext(e.into())
})?;
- let pooling = Box::new(
- Pooling::<B>::new(
- context.as_ref(),
- data_type,
- classifier_model_config.model_config.classifier_pooling.clone(),
- model_dim,
- )
- .map_err(|e| {
- eprintln!("Failed to create pooling: {:?}", e);
- Error::UnableToCreateContext(e.into())
- })?,
- );
+ let pooling = Pooling::<B>::new(
+ context.as_ref(),
+ data_type,
+ classifier_model_config.model_config.classifier_pooling.clone(),
+ model_dim,
+ )
+ .map_err(|e| {
+ eprintln!("Failed to create pooling: {:?}", e);
+ Error::UnableToCreateContext(e.into())
+ })?;
- let prediction_head = Box::new(ClassifierPredictionHead::new(
+ let prediction_head = ClassifierPredictionHead::new(
prediction_head_dense.map_err(|e| {
Error::Classifier(ClassifierError::KernelCreationFailed(format!("prediction head dense: {:?}", e)))
})?,
prediction_head_activation,
- Box::new(prediction_head_norm),
+ prediction_head_norm,
prediction_head_final_linear,
num_labels,
- ));
+ );
Ok(Self {
context,
@@ -298,11 +300,9 @@ impl<B: Backend> ClassifierContext<B> {
context: &B::Context,
data_type: DataType,
rope_type: RopeType,
- ) -> Result<Rc<Box<dyn EncodableBlock<B>>>, ClassifierError> {
- let rotation: Box<dyn EncodableBlock<B>> = Box::new(
- Rope::<B>::new(context, data_type, rope_type)
- .map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?,
- );
+ ) -> Result<Rc<Rope<B>>, ClassifierError> {
+ let rotation = Rope::<B>::new(context, data_type, rope_type)
+ .map_err(|e| ClassifierError::KernelCreationFailed(format!("RoPE: {:?}", e)))?;
Ok(Rc::new(rotation))
}
}
diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rs
index b9ea318f2..a12e4ec80 100644
--- a/crates/uzu/src/config/decoder.rs
+++ b/crates/uzu/src/config/decoder.rs
@@ -244,9 +244,10 @@ mod tests {
let config_str = r#"
{
"embedding_config": {
- "type": "QuantizedTiedEmbeddingConfig",
+ "type": "MLXQuantizedTiedEmbeddingConfig",
"input_scale": null,
"logit_soft_cap": null,
+ "group_size": 128,
"embedding_quantization_mode": "int8",
"activation_quantization_mode": "int8",
"activation_precision": "bfloat16"
@@ -347,11 +348,12 @@ mod tests {
"#;
let ground_truth_config = DecoderConfig {
- embedding_config: EmbeddingConfig::QuantizedTied {
+ embedding_config: EmbeddingConfig::MLXQuantizedTied {
common: EmbeddingConfigCommon {
input_scale: None,
logit_soft_cap: None,
},
+ group_size: 128,
embedding_quantization_mode: QuantizationMode::Int8,
activation_quantization_mode: Some(QuantizationMode::Int8),
activation_precision: ConfigDataType::BFloat16,
diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rs
index 92ce2c57c..9562a22fc 100644
--- a/crates/uzu/src/config/embedding.rs
+++ b/crates/uzu/src/config/embedding.rs
@@ -23,14 +23,6 @@ pub enum EmbeddingConfig {
common: EmbeddingConfigCommon,
precision: ConfigDataType,
},
- #[serde(rename = "QuantizedTiedEmbeddingConfig")]
- QuantizedTied {
- #[serde(flatten)]
- common: EmbeddingConfigCommon,
- embedding_quantization_mode: QuantizationMode,
- activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: ConfigDataType,
- },
#[serde(rename = "MLXQuantizedTiedEmbeddingConfig")]
MLXQuantizedTied {
#[serde(flatten)]
@@ -71,10 +63,6 @@ impl EmbeddingConfig {
common,
..
} => common,
- EmbeddingConfig::QuantizedTied {
- common,
- ..
- } => common,
EmbeddingConfig::MLXQuantizedTied {
common,
..
@@ -99,30 +87,6 @@ mod tests {
#[test]
fn test_embedding_config() {
- let config_str = r#"
- {
- "type": "QuantizedTiedEmbeddingConfig",
- "input_scale": null,
- "logit_soft_cap": null,
- "embedding_quantization_mode": "int8",
- "activation_quantization_mode": "int8",
- "activation_precision": "bfloat16"
- }
- "#;
-
- let ground_truth_config = EmbeddingConfig::QuantizedTied {
- common: EmbeddingConfigCommon {
- input_scale: None,
- logit_soft_cap: None,
- },
- embedding_quantization_mode: QuantizationMode::Int8,
- activation_quantization_mode: Some(QuantizationMode::Int8),
- activation_precision: ConfigDataType::BFloat16,
- };
-
- let deserialized_config: EmbeddingConfig = from_str(config_str).unwrap();
- assert_eq!(deserialized_config, ground_truth_config);
-
let semi_config_str = r#"
{
"type": "MLXSemiQuantizedUntiedEmbeddingConfig",
diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rs
index 11caea45b..199301464 100644
--- a/crates/uzu/src/encodable_block/activation.rs
+++ b/crates/uzu/src/encodable_block/activation.rs
@@ -2,7 +2,6 @@
use std::ops::DerefMut;
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -37,13 +36,10 @@ impl<B: Backend> Activation<B> {
output_array_id,
})
}
-}
-impl<B: Backend> EncodableBlock<B> for Activation<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index 1a8bf6bf6..e2ab1f4dc 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -14,7 +14,7 @@ use crate::{
attention::{AttentionGemmArguments, AttentionGemmBlock},
},
},
- encodable_block::{EncodableBlock, EncodingParameters},
+ encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState, HashMapId},
};
@@ -133,10 +133,8 @@ impl<B: Backend> Attention<B> {
KernelVariant::SinglePass
}
-}
-impl<B: Backend> EncodableBlock<B> for Attention<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index 33b3ca455..9c1d60ac5 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -7,8 +7,7 @@ use crate::{
backends::common::{Backend, CommandBuffer},
config::TransformerLayerConfig,
encodable_block::{
- Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,
- transformer_layer::{linear_block, mlp_block},
+ Attention, EncodingParameters, Linear, Mlp, Normalization, QKNorm, Rope, TensorAddSwap, TensorCopy,
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -17,20 +16,20 @@ use crate::{
pub struct ClassifierLayer<B: Backend> {
#[cfg_attr(not(feature = "tracing"), allow(dead_code))]
layer_index: usize,
- copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>>,
- pre_attention_norm: Option<Box<dyn EncodableBlock<B>>>,
- qkv_projection: Box<dyn EncodableBlock<B>>,
- qk_norm: Option<Box<dyn EncodableBlock<B>>>,
- rope: Rc<Box<dyn EncodableBlock<B>>>,
- attention: Box<dyn EncodableBlock<B>>,
- out_projection: Box<dyn EncodableBlock<B>>,
- post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,
- mixer_residual_add: Box<dyn EncodableBlock<B>>,
- copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>>,
- pre_mlp_norm: Box<dyn EncodableBlock<B>>,
- mlp: Box<dyn EncodableBlock<B>>,
- post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,
- mlp_residual_add: Box<dyn EncodableBlock<B>>,
+ copy_main_to_shortcut_mixer: TensorCopy<B>,
+ pre_attention_norm: Option<Normalization<B>>,
+ qkv_projection: Box<dyn Linear<B>>,
+ qk_norm: Option<QKNorm<B>>,
+ rope: Rc<Rope<B>>,
+ attention: Attention<B>,
+ out_projection: Box<dyn Linear<B>>,
+ post_attention_norm: Option<Normalization<B>>,
+ mixer_residual_add: TensorAddSwap<B>,
+ copy_main_to_shortcut_mlp: TensorCopy<B>,
+ pre_mlp_norm: Normalization<B>,
+ mlp: Box<dyn Mlp<B>>,
+ post_mlp_norm: Option<Normalization<B>>,
+ mlp_residual_add: TensorAddSwap<B>,
}
impl<B: Backend> ClassifierLayer<B> {
@@ -45,7 +44,7 @@ impl<B: Backend> ClassifierLayer<B> {
num_groups: usize,
attention_scale: Option<f32>,
layer_loader: &ParameterTree<B::Context>,
- rope: Rc<Box<dyn EncodableBlock<B>>>,
+ rope: Rc<Rope<B>>,
) -> Self {
autoreleasepool(|_| {
let ctx = context.as_ref(); // Reference for functions expecting &B::Context
@@ -53,37 +52,34 @@ impl<B: Backend> ClassifierLayer<B> {
layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
- let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>> = Box::new(
- TensorCopy::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap(),
- );
+ let copy_main_to_shortcut_mixer = TensorCopy::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap();
- let pre_attention_norm: Option<Box<dyn EncodableBlock<B>>> =
- if let Some(norm_config) = &layer_config.pre_attention_norm_config {
- if layer_loader.subtree("pre_mixer_norm").is_ok() {
- Some(Box::new(
- Normalization::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &layer_loader.subtree("pre_mixer_norm").unwrap(),
- )
- .expect("Failed to create pre-attention norm kernel"),
- ))
- } else {
- None
- }
+ let pre_attention_norm = if let Some(norm_config) = &layer_config.pre_attention_norm_config {
+ if layer_loader.subtree("pre_mixer_norm").is_ok() {
+ Some(
+ Normalization::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("pre_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create pre-attention norm kernel"),
+ )
} else {
None
- };
+ }
+ } else {
+ None
+ };
- let qkv_projection = linear_block(
+ let qkv_projection = <dyn Linear<B>>::new(
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
@@ -95,27 +91,27 @@ impl<B: Backend> ClassifierLayer<B> {
)
.expect("Failed to create qkv projection");
- let qk_norm: Option<Box<dyn EncodableBlock<B>>> =
- if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
- match QKNorm::new(
- ctx,
- intermediate_data_type,
- attention_config.query_norm_config.clone(),
- attention_config.key_norm_config.clone(),
- ArrayId::QKV,
- &layer_loader.subtree("mixer").unwrap(),
- num_heads,
- num_groups,
- head_dim,
- ) {
- Ok(norm) => Some(Box::new(norm)),
- Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
- }
- } else {
- None
- };
+ let qk_norm = if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some()
+ {
+ match QKNorm::new(
+ ctx,
+ intermediate_data_type,
+ attention_config.query_norm_config.clone(),
+ attention_config.key_norm_config.clone(),
+ ArrayId::QKV,
+ &layer_loader.subtree("mixer").unwrap(),
+ num_heads,
+ num_groups,
+ head_dim,
+ ) {
+ Ok(norm) => Some(norm),
+ Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
+ }
+ } else {
+ None
+ };
- let out_projection = linear_block(
+ let out_projection = <dyn Linear<B>>::new(
&attention_config.out_projection_config,
attention_config.has_out_biases,
num_heads * head_dim,
@@ -127,54 +123,47 @@ impl<B: Backend> ClassifierLayer<B> {
)
.expect("Failed to create out projection");
- let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =
- if let Some(norm_config) = &layer_config.post_attention_norm_config {
- Some(Box::new(
- Normalization::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &layer_loader.subtree("post_mixer_norm").unwrap(),
- )
- .expect("Failed to create post-attention norm kernel"),
- ))
- } else {
- None
- };
-
- let mixer_residual_add: Box<dyn EncodableBlock<B>> = Box::new(
- TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
+ Some(
+ Normalization::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("post_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create post-attention norm kernel"),
)
- .unwrap(),
- );
+ } else {
+ None
+ };
- let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>> = Box::new(
- TensorCopy::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap(),
- );
+ let mixer_residual_add = TensorAddSwap::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap();
- let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(
- Normalization::new(
- ctx,
- intermediate_data_type,
- layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &layer_loader.subtree("pre_mlp_norm").unwrap(),
- )
- .expect("Failed to create pre-MLP norm kernel"),
- );
+ let copy_main_to_shortcut_mlp = TensorCopy::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap();
+
+ let pre_mlp_norm = Normalization::new(
+ ctx,
+ intermediate_data_type,
+ layer_config.pre_mlp_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("pre_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create pre-MLP norm kernel");
- let mlp = mlp_block(
+ let mlp = <dyn Mlp<B>>::new(
&layer_config.mlp_config,
model_dim,
hidden_dim,
@@ -183,44 +172,39 @@ impl<B: Backend> ClassifierLayer<B> {
)
.expect("Failed to create mlp block");
- let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =
- if let Some(norm_config) = &layer_config.post_mlp_norm_config {
- Some(Box::new(
- Normalization::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &layer_loader.subtree("post_mlp_norm").unwrap(),
- )
- .expect("Failed to create post-MLP norm kernel"),
- ))
- } else {
- None
- };
-
- let attention: Box<dyn EncodableBlock<B>> = Box::new(
- Attention::new(
- ctx,
- intermediate_data_type,
- layer_index,
- attention_scale,
- attention_config.has_sinks,
- false,
- attention_config.sliding_window_size,
+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ Some(
+ Normalization::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &layer_loader.subtree("post_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create post-MLP norm kernel"),
)
- .expect("Failed to create attention kernel"),
- );
+ } else {
+ None
+ };
- let mlp_residual_add: Box<dyn EncodableBlock<B>> = Box::new(
- TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
- )
- .unwrap(),
- );
+ let attention = Attention::new(
+ ctx,
+ intermediate_data_type,
+ layer_index,
+ attention_scale,
+ attention_config.has_sinks,
+ false,
+ attention_config.sliding_window_size,
+ )
+ .expect("Failed to create attention kernel");
+
+ let mlp_residual_add = TensorAddSwap::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap();
Self {
layer_index,
@@ -241,10 +225,8 @@ impl<B: Backend> ClassifierLayer<B> {
}
})
}
-}
-impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
@@ -258,10 +240,10 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());
}
- self.copy_main_to_shortcut_mixer.encode(state, parameters, command_buffer)?;
+ self.copy_main_to_shortcut_mixer.encode(state, command_buffer)?;
if let Some(ref pre_attn_norm) = self.pre_attention_norm {
- pre_attn_norm.encode(state, parameters, command_buffer)?;
+ pre_attn_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(
@@ -281,20 +263,20 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
}
}
- self.qkv_projection.encode(state, parameters, command_buffer)?;
+ self.qkv_projection.encode(state, command_buffer)?;
if let Some(ref qk_norm) = self.qk_norm {
- qk_norm.encode(state, parameters, command_buffer)?;
+ qk_norm.encode(state, command_buffer)?;
}
- self.rope.encode(state, parameters, command_buffer)?;
+ self.rope.encode(state, command_buffer)?;
self.attention.encode(state, parameters, command_buffer)?;
- self.out_projection.encode(state, parameters, command_buffer)?;
+ self.out_projection.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
}
if let Some(ref post_attn_norm) = self.post_attention_norm {
- post_attn_norm.encode(state, parameters, command_buffer)?;
+ post_attn_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(
@@ -305,41 +287,40 @@ impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {
}
}
- self.mixer_residual_add.encode(state, parameters, command_buffer)?;
+ self.mixer_residual_add.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
}
- self.copy_main_to_shortcut_mlp.encode(state, parameters, command_buffer)?;
+ self.copy_main_to_shortcut_mlp.encode(state, command_buffer)?;
- self.pre_mlp_norm.encode(state, parameters, command_buffer)?;
+ self.pre_mlp_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
}
- self.mlp.encode(state, parameters, command_buffer)?;
+ self.mlp.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());
}
if let Some(ref post_mlp_norm) = self.post_mlp_norm {
- post_mlp_norm.encode(state, parameters, command_buffer)?;
+ post_mlp_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
}
}
- self.mlp_residual_add.encode(state, parameters, command_buffer)?;
+ self.mlp_residual_add.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());
}
- let _ = parameters;
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index 75b4338c7..890b474a9 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -2,14 +2,13 @@
use std::rc::Rc;
+use thiserror::Error;
+
use crate::{
DataType, DecoderConfig,
backends::common::{Backend, CommandBuffer},
config::{DecoderLayerType, MixerConfig},
- encodable_block::{
- EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,
- transformer_layer::{embed_block, readout_block},
- },
+ encodable_block::{Embedding, EncodingParameters, LayerExecutables, RMSNorm, Rope, embedding::EmbeddingError},
forward_pass::{
model_shape::ModelShape,
state::{ArrayId, ForwardPassState, RopeType},
@@ -17,14 +16,21 @@ use crate::{
parameters::ParameterTree,
};
+#[derive(Debug, Error)]
+pub enum DecoderError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Embedding error: {0}")]
+ EmbeddingError(#[from] EmbeddingError<B>),
+}
+
/// Full decoder executable with all layers and components.
pub struct Decoder<B: Backend> {
- pub embed: Box<dyn EncodableBlock<B>>,
+ pub embed: Embedding<B>,
pub layers: Box<[LayerExecutables<B>]>,
- pub norm: Box<dyn EncodableBlock<B>>,
- pub readout: Box<dyn EncodableBlock<B>>,
- pub global_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
- pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
+ pub norm: RMSNorm<B>,
+ pub global_rope: Option<Rc<Rope<B>>>,
+ pub local_rope: Option<Rc<Rope<B>>>,
}
impl<B: Backend> Decoder<B> {
@@ -35,9 +41,14 @@ impl<B: Backend> Decoder<B> {
) -> Self {
let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");
- let embed = embed_block(&decoder_config, context.as_ref(), root_weight_loader);
-
- let readout = readout_block(&decoder_config, context.as_ref(), root_weight_loader);
+ let embed = Embedding::new(
+ context.as_ref(),
+ decoder_config.vocab_size as u32,
+ decoder_config.model_dim as u32,
+ &decoder_config.embedding_config,
+ &root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree"),
+ )
+ .expect("Failed to create embedding");
let attention_data_type = Self::attention_data_type(&decoder_config);
let norm_reference_layer =
@@ -114,24 +125,21 @@ impl<B: Backend> Decoder<B> {
})
.collect::<Vec<_>>();
- let norm_block: Box<dyn EncodableBlock<B>> = Box::new(
- RMSNorm::new(
- context.as_ref(),
- norm_data_type,
- decoder_config.output_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_weight_loader.subtree("output_norm").unwrap(),
- )
- .map(RMSNorm::with_sampling_range)
- .expect("Failed to create output RMS norm kernel"),
- );
+ let norm_block = RMSNorm::new(
+ context.as_ref(),
+ norm_data_type,
+ decoder_config.output_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_weight_loader.subtree("output_norm").unwrap(),
+ )
+ .map(RMSNorm::with_sampling_range)
+ .expect("Failed to create output RMS norm kernel");
Self {
embed,
layers: layers.into_boxed_slice(),
norm: norm_block,
- readout,
global_rope,
local_rope,
}
@@ -141,10 +149,8 @@ impl<B: Backend> Decoder<B> {
context: &B::Context,
data_type: DataType,
rope_type: RopeType,
- ) -> Rc<Box<dyn EncodableBlock<B>>> {
- let rotation: Box<dyn EncodableBlock<B>> =
- Box::new(Rope::<B>::new(context, data_type, rope_type).expect("Failed to create Rope"));
- Rc::new(rotation)
+ ) -> Rc<Rope<B>> {
+ Rc::new(Rope::<B>::new(context, data_type, rope_type).expect("Failed to create Rope"))
}
fn attention_data_type(decoder_config: &DecoderConfig) -> Option<DataType> {
@@ -159,33 +165,31 @@ impl<B: Backend> Decoder<B> {
.map(|attention_config| attention_config.qkv_projection_config.activation_precision().into())
})
}
-}
-impl<B: Backend> EncodableBlock<B> for Decoder<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- self.embed.encode(state, parameters, command_buffer)?;
+ ) -> Result<(), DecoderError<B>> {
+ self.embed.encode_lookup(state, command_buffer)?;
for layer in self.layers.iter() {
- layer.encode(state, parameters, command_buffer)?;
+ layer.encode(state, parameters, command_buffer).map_err(DecoderError::BackendError)?;
}
if state.is_prefilling() {
return Ok(());
}
- self.norm.encode(state, parameters, command_buffer)?;
+ self.norm.encode(state, command_buffer).map_err(DecoderError::BackendError)?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
state.encode_copy_array(command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());
}
- self.readout.encode(state, parameters, command_buffer)?;
+ self.embed.encode_readout(state, command_buffer)?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
new file mode 100644
index 000000000..4e6deb496
--- /dev/null
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -0,0 +1,611 @@
+use std::{
+ cell::RefCell,
+ ops::{Deref, DerefMut},
+};
+
+use thiserror::Error;
+
+use crate::{
+ DataType, EmbeddingConfig, QuantizationMode,
+ backends::common::{
+ Backend, CommandBuffer, Kernels,
+ kernel::{
+ FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,
+ matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},
+ quant_matmul::{
+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
+ QuantizedMatmulKernelEncodable, QuantizedMatmulType,
+ },
+ },
+ },
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLeaf, ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum EmbeddingError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Matmul error: {0}")]
+ MatmulError(#[from] MatmulError<B>),
+ #[error("QuantizedMatmul error: {0}")]
+ QuantizedMatmulError(#[from] QuantizedMatmulError<B>),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(#[from] ParameterLoaderError<B>),
+ #[error("Unsupported configuration: {0}")]
+ UnsupportedConfiguration(String),
+ #[error("Invalid tensor: got {shape:?} @ {data_type:?}, expected {expected_shape:?} @ {expected_data_type:?}")]
+ InvalidTensor {
+ shape: Box<[usize]>,
+ data_type: DataType,
+ expected_shape: Box<[usize]>,
+ expected_data_type: DataType,
+ },
+}
+
+enum TiedEmbeddingType<B: Backend> {
+ FullPrecision {
+ weights: B::Buffer,
+ lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
+ readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
+ },
+ Quantized {
+ weights: B::Buffer,
+ scales: B::Buffer,
+ biases: B::Buffer,
+ lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
+ readout: QuantizedMatmulKernelEncodable<B>,
+ },
+}
+
+enum UntiedEmbeddingLookupType<B: Backend> {
+ FullPrecision {
+ weights: B::Buffer,
+ lookup: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
+ },
+ Quantized {
+ weights: B::Buffer,
+ scales: B::Buffer,
+ biases: B::Buffer,
+ lookup: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
+ },
+}
+
+enum UntiedEmbeddingReadoutType<B: Backend> {
+ FullPrecision {
+ weights: B::Buffer,
+ readout: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
+ },
+ Quantized {
+ weights: B::Buffer,
+ scales: B::Buffer,
+ biases: B::Buffer,
+ readout: QuantizedMatmulKernelEncodable<B>,
+ },
+}
+
+enum EmbeddingTying<B: Backend> {
+ Tied {
+ ty: TiedEmbeddingType<B>,
+ },
+ Untied {
+ input_ty: UntiedEmbeddingLookupType<B>,
+ output_ty: UntiedEmbeddingReadoutType<B>,
+ },
+}
+
+pub struct Embedding<B: Backend> {
+ tying: EmbeddingTying<B>,
+ input_scale: f32,
+ vocab_size: u32,
+ model_dim: u32,
+}
+
+fn quant_mode_to_int(quant_mode: QuantizationMode) -> u32 {
+ match quant_mode {
+ QuantizationMode::UInt4 => 0,
+ QuantizationMode::Int8 => 1,
+ QuantizationMode::UInt8 => 2,
+ }
+}
+
+fn validate_tensor<'file, 'context, 'leaf, B: Backend>(
+ weights_leaf: &ParameterLeaf<'file, 'context, 'leaf, B::Context>,
+ expected_shape: [usize; 2],
+ expected_data_type: DataType,
+) -> Result<(), EmbeddingError<B>> {
+ let shape = weights_leaf.shape();
+ let data_type = weights_leaf.data_type();
+
+ if (shape, data_type) != (expected_shape.as_ref(), expected_data_type) {
+ return Err(EmbeddingError::InvalidTensor {
+ shape: shape.into(),
+ data_type: weights_leaf.data_type(),
+ expected_shape: expected_shape.into(),
+ expected_data_type,
+ });
+ }
+
+ Ok(())
+}
+
+impl<B: Backend> Embedding<B> {
+ pub fn new(
+ context: &B::Context,
+ vocab_size: u32,
+ model_dim: u32,
+ config: &EmbeddingConfig,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, EmbeddingError<B>> {
+ let common = config.common();
+
+ let tying = match config {
+ EmbeddingConfig::Tied {
+ common: _,
+ precision,
+ } => {
+ let data_type = (*precision).into();
+
+ let weights_leaf = parameter_tree.leaf("weights")?;
+ validate_tensor(&weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
+
+ let weights = weights_leaf.read_buffer()?;
+
+ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?;
+ let readout =
+ RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);
+
+ EmbeddingTying::Tied {
+ ty: TiedEmbeddingType::FullPrecision {
+ weights,
+ lookup,
+ readout,
+ },
+ }
+ },
+ EmbeddingConfig::Untied {
+ common: _,
+ precision,
+ } => {
+ let data_type = (*precision).into();
+
+ let input_weights_leaf = parameter_tree.leaf("input_weights")?;
+ validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
+ let output_weights_leaf = parameter_tree.leaf("output_weights")?;
+ validate_tensor(&output_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
+
+ let input_weights = input_weights_leaf.read_buffer()?;
+ let output_weights = output_weights_leaf.read_buffer()?;
+
+ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?;
+ let readout =
+ RefCell::new(<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?);
+
+ EmbeddingTying::Untied {
+ input_ty: UntiedEmbeddingLookupType::FullPrecision {
+ weights: input_weights,
+ lookup,
+ },
+ output_ty: UntiedEmbeddingReadoutType::FullPrecision {
+ weights: output_weights,
+ readout,
+ },
+ }
+ },
+ EmbeddingConfig::MLXQuantizedTied {
+ common: _,
+ group_size,
+ embedding_quantization_mode,
+ activation_quantization_mode,
+ activation_precision,
+ } => {
+ let data_type = (*activation_precision).into();
+
+ let packing_divisor = embedding_quantization_mode.packing_divisor();
+ let storage_data_type = embedding_quantization_mode.storage_type();
+
+ let num_groups = (model_dim as usize).div_ceil(*group_size);
+
+ let weights_leaf = parameter_tree.leaf("weights")?;
+ validate_tensor(
+ &weights_leaf,
+ [vocab_size as usize, model_dim as usize / packing_divisor],
+ storage_data_type,
+ )?;
+ let scales_leaf = parameter_tree.leaf("scales")?;
+ validate_tensor(&scales_leaf, [vocab_size as usize, num_groups], data_type)?;
+ let biases_leaf = parameter_tree.leaf("biases")?;
+ validate_tensor(&biases_leaf, [vocab_size as usize, num_groups], data_type)?;
+
+ let weights = weights_leaf.read_buffer()?;
+ let scales = scales_leaf.read_buffer()?;
+ let biases = biases_leaf.read_buffer()?;
+
+ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ context,
+ data_type,
+ *group_size as u32,
+ quant_mode_to_int(*embedding_quantization_mode),
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let readout = QuantizedMatmulKernelEncodable::new(
+ context,
+ QuantizedMatmulConfiguration {
+ data_type,
+ group_size: *group_size,
+ input_dim: model_dim as usize,
+ output_dim: vocab_size as usize,
+ mode: *embedding_quantization_mode,
+ quantization_type: QuantizedMatmulType::Mlx,
+ weights_transposed: true,
+ },
+ )?;
+
+ if let Some(activation_quantization_mode) = activation_quantization_mode {
+ return Err(EmbeddingError::UnsupportedConfiguration(format!(
+ "activation_quantization_mode={activation_quantization_mode:?}"
+ )));
+ }
+
+ EmbeddingTying::Tied {
+ ty: TiedEmbeddingType::Quantized {
+ weights,
+ scales,
+ biases,
+ lookup,
+ readout,
+ },
+ }
+ },
+ EmbeddingConfig::MLXQuantizedUntied {
+ common: _,
+ group_size,
+ embedding_quantization_mode,
+ activation_quantization_mode,
+ activation_precision,
+ } => {
+ let data_type = (*activation_precision).into();
+
+ let packing_divisor = embedding_quantization_mode.packing_divisor();
+ let storage_data_type = embedding_quantization_mode.storage_type();
+
+ let num_groups = (model_dim as usize).div_ceil(*group_size);
+
+ let input_weights_leaf = parameter_tree.leaf("input_weights")?;
+ validate_tensor(
+ &input_weights_leaf,
+ [vocab_size as usize, model_dim as usize / packing_divisor],
+ storage_data_type,
+ )?;
+ let input_scales_leaf = parameter_tree.leaf("input_scales")?;
+ validate_tensor(&input_scales_leaf, [vocab_size as usize, num_groups], data_type)?;
+ let input_biases_leaf = parameter_tree.leaf("input_biases")?;
+ validate_tensor(&input_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
+
+ let output_weights_leaf = parameter_tree.leaf("output_weights")?;
+ validate_tensor(
+ &output_weights_leaf,
+ [vocab_size as usize, model_dim as usize / packing_divisor],
+ storage_data_type,
+ )?;
+ let output_scales_leaf = parameter_tree.leaf("output_scales")?;
+ validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;
+ let output_biases_leaf = parameter_tree.leaf("output_biases")?;
+ validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
+
+ let input_weights = input_weights_leaf.read_buffer()?;
+ let input_scales = input_scales_leaf.read_buffer()?;
+ let input_biases = input_biases_leaf.read_buffer()?;
+
+ let output_weights = output_weights_leaf.read_buffer()?;
+ let output_scales = output_scales_leaf.read_buffer()?;
+ let output_biases = output_biases_leaf.read_buffer()?;
+
+ let lookup = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(
+ context,
+ data_type,
+ *group_size as u32,
+ quant_mode_to_int(*embedding_quantization_mode),
+ )
+ .map_err(EmbeddingError::BackendError)?;
+ let readout = QuantizedMatmulKernelEncodable::new(
+ context,
+ QuantizedMatmulConfiguration {
+ data_type,
+ group_size: *group_size,
+ input_dim: model_dim as usize,
+ output_dim: vocab_size as usize,
+ mode: *embedding_quantization_mode,
+ quantization_type: QuantizedMatmulType::Mlx,
+ weights_transposed: true,
+ },
+ )?;
+
+ if let Some(activation_quantization_mode) = activation_quantization_mode {
+ return Err(EmbeddingError::UnsupportedConfiguration(format!(
+ "activation_quantization_mode={activation_quantization_mode:?}"
+ )));
+ }
+
+ EmbeddingTying::Untied {
+ input_ty: UntiedEmbeddingLookupType::Quantized {
+ weights: input_weights,
+ scales: input_scales,
+ biases: input_biases,
+ lookup,
+ },
+ output_ty: UntiedEmbeddingReadoutType::Quantized {
+ weights: output_weights,
+ scales: output_scales,
+ biases: output_biases,
+ readout,
+ },
+ }
+ },
+ EmbeddingConfig::MLXSemiQuantizedUntied {
+ common: _,
+ group_size,
+ embedding_quantization_mode,
+ activation_quantization_mode,
+ activation_precision,
+ } => {
+ let data_type = (*activation_precision).into();
+
+ let packing_divisor = embedding_quantization_mode.packing_divisor();
+ let storage_data_type = embedding_quantization_mode.storage_type();
+
+ let num_groups = (model_dim as usize).div_ceil(*group_size);
+
+ let input_weights_leaf = parameter_tree.leaf("input_weights")?;
+ validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
+
+ let output_weights_leaf = parameter_tree.leaf("output_weights")?;
+ validate_tensor(
+ &output_weights_leaf,
+ [vocab_size as usize, model_dim as usize / packing_divisor],
+ storage_data_type,
+ )?;
+ let output_scales_leaf = parameter_tree.leaf("output_scales")?;
+ validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;
+ let output_biases_leaf = parameter_tree.leaf("output_biases")?;
+ validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
+
+ let input_weights = input_weights_leaf.read_buffer()?;
+
+ let output_weights = output_weights_leaf.read_buffer()?;
+ let output_scales = output_scales_leaf.read_buffer()?;
+ let output_biases = output_biases_leaf.read_buffer()?;
+
+ let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
+ .map_err(EmbeddingError::BackendError)?;
+ let readout = QuantizedMatmulKernelEncodable::new(
+ context,
+ QuantizedMatmulConfiguration {
+ data_type,
+ group_size: *group_size,
+ input_dim: model_dim as usize,
+ output_dim: vocab_size as usize,
+ mode: *embedding_quantization_mode,
+ quantization_type: QuantizedMatmulType::Mlx,
+ weights_transposed: true,
+ },
+ )?;
+
+ if let Some(activation_quantization_mode) = activation_quantization_mode {
+ return Err(EmbeddingError::UnsupportedConfiguration(format!(
+ "activation_quantization_mode={activation_quantization_mode:?}"
+ )));
+ }
+
+ EmbeddingTying::Untied {
+ input_ty: UntiedEmbeddingLookupType::FullPrecision {
+ weights: input_weights,
+ lookup,
+ },
+ output_ty: UntiedEmbeddingReadoutType::Quantized {
+ weights: output_weights,
+ scales: output_scales,
+ biases: output_biases,
+ readout,
+ },
+ }
+ },
+ };
+
+ let input_scale = common.input_scale.unwrap_or(1.0);
+
+ if let Some(logit_soft_cap) = common.logit_soft_cap {
+ return Err(EmbeddingError::UnsupportedConfiguration(format!("logit_soft_cap={logit_soft_cap:?}")));
+ }
+
+ Ok(Self {
+ tying,
+ input_scale,
+ vocab_size,
+ model_dim,
+ })
+ }
+
+ pub fn encode_lookup(
+ &self,
+ state: &mut ForwardPassState<B>,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), EmbeddingError<B>> {
+ let batch_size = state.active_suffix_length() as u32;
+
+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
+
+ let token_ids_array = arrays[0].borrow();
+ let token_ids_buffer_rc = token_ids_array.buffer();
+ let token_ids_buffer_borrow = token_ids_buffer_rc.borrow();
+ let token_ids = token_ids_buffer_borrow.deref();
+
+ let output_array = arrays[1].borrow();
+ let output_buffer_rc = output_array.buffer();
+ let mut output_buffer_borrow = output_buffer_rc.borrow_mut();
+ let output = output_buffer_borrow.deref_mut();
+
+ match &self.tying {
+ EmbeddingTying::Tied {
+ ty:
+ TiedEmbeddingType::FullPrecision {
+ weights,
+ lookup,
+ readout: _,
+ },
+ }
+ | EmbeddingTying::Untied {
+ input_ty:
+ UntiedEmbeddingLookupType::FullPrecision {
+ weights,
+ lookup,
+ },
+ output_ty: _,
+ } => lookup.encode(
+ token_ids,
+ weights,
+ output,
+ batch_size,
+ self.vocab_size,
+ self.model_dim,
+ self.input_scale,
+ command_buffer,
+ ),
+ EmbeddingTying::Tied {
+ ty:
+ TiedEmbeddingType::Quantized {
+ weights,
+ scales,
+ biases,
+ lookup,
+ readout: _,
+ },
+ }
+ | EmbeddingTying::Untied {
+ input_ty:
+ UntiedEmbeddingLookupType::Quantized {
+ weights,
+ scales,
+ biases,
+ lookup,
+ },
+ output_ty: _,
+ } => {
+ lookup.encode(
+ token_ids,
+ weights,
+ scales,
+ biases,
+ output,
+ batch_size,
+ self.vocab_size,
+ self.model_dim,
+ self.input_scale,
+ command_buffer,
+ );
+ },
+ };
+
+ Ok(())
+ }
+
+ pub fn encode_readout(
+ &self,
+ state: &mut ForwardPassState<B>,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), EmbeddingError<B>> {
+ let batch_size = state.sampling_length();
+
+ if batch_size == 0 {
+ return Ok(());
+ }
+
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
+
+ let input_array = arrays[0].borrow();
+ let input_buffer_rc = input_array.buffer();
+ let input_buffer_borrow = input_buffer_rc.borrow();
+ let input = input_buffer_borrow.deref();
+
+ let output_array = arrays[1].borrow_mut();
+ let output_buffer_rc = output_array.buffer();
+ let mut output_buffer_borrow = output_buffer_rc.borrow_mut();
+ let output = output_buffer_borrow.deref_mut();
+
+ let input_offset = state.sampling_start() * self.model_dim as usize * input_array.data_type().size_in_bytes();
+
+ match &self.tying {
+ EmbeddingTying::Tied {
+ ty:
+ TiedEmbeddingType::FullPrecision {
+ weights,
+ lookup: _,
+ readout,
+ },
+ }
+ | EmbeddingTying::Untied {
+ input_ty: _,
+ output_ty:
+ UntiedEmbeddingReadoutType::FullPrecision {
+ weights,
+ readout,
+ },
+ } => {
+ readout.borrow_mut().encode(
+ state.context(),
+ command_buffer,
+ FullPrecisionMatmulArguments {
+ a: input,
+ a_offset: input_offset,
+ b: weights,
+ output,
+ bias: None,
+ batch: batch_size,
+ input_dim: self.model_dim as usize,
+ output_dim: self.vocab_size as usize,
+ },
+ );
+ },
+ EmbeddingTying::Tied {
+ ty:
+ TiedEmbeddingType::Quantized {
+ weights,
+ scales,
+ biases,
+ lookup: _,
+ readout,
+ },
+ }
+ | EmbeddingTying::Untied {
+ input_ty: _,
+ output_ty:
+ UntiedEmbeddingReadoutType::Quantized {
+ weights,
+ scales,
+ biases,
+ readout,
+ },
+ } => {
+ readout.encode(
+ command_buffer,
+ QuantizedMatmulArguments {
+ a_buffer: input,
+ a_offset: input_offset,
+ b_buffer: weights,
+ scales_buffer: scales,
+ zero_points_or_biases_buffer: biases,
+ output_buffer: output,
+ batch: batch_size,
+ input_dim: self.model_dim as usize,
+ output_dim: self.vocab_size as usize,
+ quantization_type: QuantizedMatmulType::Mlx,
+ },
+ )?;
+ },
+ };
+
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
deleted file mode 100644
index c9e0fc151..000000000
--- a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
+++ /dev/null
@@ -1,117 +0,0 @@
-//! Full precision embedding lookup encodable.
-
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
-use thiserror::Error;
-
-use super::{EncodableBlock, EncodingParameters};
-use crate::{
- DataType,
- backends::common::{
- Backend, CommandBuffer,
- kernel::{FullPrecisionEmbeddingLookupKernel, Kernels},
- },
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-#[derive(Debug, Error)]
-pub enum FullPrecisionEmbeddingLookupError<B: Backend> {
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
- #[error(
- "Embedding lookup weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"
- )]
- InvalidWeightsShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_model_dim: usize,
- },
- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidWeightsDataType {
- got: DataType,
- expected: DataType,
- },
-}
-
-pub struct FullPrecisionEmbeddingLookup<B: Backend> {
- kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
- weights_buffer: Rc<RefCell<B::Buffer>>,
- vocab_size: u32,
- model_dim: u32,
- input_scale: f32,
-}
-
-impl<B: Backend> FullPrecisionEmbeddingLookup<B> {
- pub fn new(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- input_scale: Option<f32>,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, FullPrecisionEmbeddingLookupError<B>> {
- let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
- .map_err(FullPrecisionEmbeddingLookupError::BackendError)?;
-
- let weights = parameter_tree
- .leaf("weights")
- .or_else(|_| parameter_tree.leaf("input_weights"))
- .map_err(FullPrecisionEmbeddingLookupError::ParameterError)?;
-
- if weights.shape() != [vocab_size, model_dim] {
- return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsShape {
- got: weights.shape().to_vec().into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_model_dim: model_dim,
- });
- }
-
- if weights.data_type() != data_type {
- return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsDataType {
- got: weights.data_type(),
- expected: data_type,
- });
- }
-
- Ok(Self {
- kernel,
- weights_buffer: weights.buffer(),
- vocab_size: vocab_size as u32,
- model_dim: model_dim as u32,
- input_scale: input_scale.unwrap_or(1.0),
- })
- }
-}
-
-impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {
- fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
- let batch_size = state.active_suffix_length();
- let token_ids_array = arrays[0].borrow_mut();
- let output_array = arrays[1].borrow_mut();
-
- self.kernel.encode(
- token_ids_array.buffer().borrow().deref(),
- self.weights_buffer.borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- batch_size as u32,
- self.vocab_size,
- self.model_dim,
- self.input_scale,
- command_buffer,
- );
- Ok(())
- }
-}
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
deleted file mode 100644
index d993926f0..000000000
--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
+++ /dev/null
@@ -1,130 +0,0 @@
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
-use thiserror::Error;
-
-use super::{EncodableBlock, EncodingParameters};
-use crate::{
- DataType,
- backends::common::{
- Backend, CommandBuffer,
- kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulError, MatmulKernels},
- },
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-#[derive(Debug, Error)]
-pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {
- #[error("Matmul error: {0}")]
- MatmulError(#[from] MatmulError<B>),
- #[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
- #[error(
- "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"
- )]
- InvalidWeightsShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_model_dim: usize,
- },
- #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidWeightsDataType {
- expected: DataType,
- got: DataType,
- },
-}
-
-pub struct FullPrecisionEmbeddingReadout<B: Backend> {
- kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
- weights_buffer: Rc<RefCell<B::Buffer>>,
- vocab_size: usize,
- model_dim: usize,
-}
-
-impl<B: Backend> FullPrecisionEmbeddingReadout<B> {
- pub fn new(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, FullPrecisionEmbeddingReadoutError<B>> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(FullPrecisionEmbeddingReadoutError::UnsupportedDataType(data_type));
- }
-
- let weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => {
- parameter_tree.leaf("output_weights").map_err(FullPrecisionEmbeddingReadoutError::ParameterError)?
- },
- };
-
- let weights_shape = weights.shape().to_vec();
- if weights_shape != [vocab_size, model_dim] {
- return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsShape {
- got: weights_shape.into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_model_dim: model_dim,
- });
- }
-
- if weights.data_type() != data_type {
- return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsDataType {
- expected: data_type,
- got: weights.data_type(),
- });
- }
-
- let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)?;
-
- Ok(Self {
- kernel: RefCell::new(kernel),
- weights_buffer: weights.buffer(),
- vocab_size,
- model_dim,
- })
- }
-}
-
-impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {
- fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.sampling_length();
- if batch_size == 0 {
- return Ok(());
- }
-
- let sampling_start = state.sampling_start();
- let input_array = arrays[0].borrow_mut();
- let output_array = arrays[1].borrow_mut();
- let element_size = input_array.data_type().size_in_bytes();
-
- self.kernel.borrow_mut().encode(
- state.context(),
- command_buffer,
- FullPrecisionMatmulArguments {
- a: input_array.buffer().borrow().deref(),
- a_offset: sampling_start * self.model_dim * element_size,
- b: self.weights_buffer.borrow().deref(),
- output: output_array.buffer().borrow_mut().deref_mut(),
- bias: None,
- batch: batch_size,
- input_dim: self.model_dim,
- output_dim: self.vocab_size,
- },
- );
- Ok(())
- }
-}
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index b51a5d85c..0c4b4aed5 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -10,9 +10,8 @@ use crate::{
backends::common::{Backend, CommandBuffer},
config::{DecoderLayerType, MixerConfig},
encodable_block::{
- Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,
+ Attention, EncodingParameters, Linear, MambaMixer, Mlp, QKNorm, RMSNorm, Rope, ShortConvMixer, TensorAddSwap,
TensorCopy,
- transformer_layer::{linear_block, mlp_block},
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -21,14 +20,14 @@ use crate::{
/// A single decoder layer with all its components.
pub struct LayerExecutables<B: Backend> {
pub layer_index: usize,
- pub copy_main_to_shortcut: Box<dyn EncodableBlock<B>>,
- pub pre_attention_norm: Box<dyn EncodableBlock<B>>,
+ pub copy_main_to_shortcut: TensorCopy<B>,
+ pub pre_attention_norm: RMSNorm<B>,
pub(crate) mixer: MixerExecutables<B>,
- pub post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,
- pub main_shortcut_add_swap: Box<dyn EncodableBlock<B>>,
- pub pre_mlp_norm: Box<dyn EncodableBlock<B>>,
- pub mlp: Box<dyn EncodableBlock<B>>,
- pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,
+ pub post_attention_norm: Option<RMSNorm<B>>,
+ pub main_shortcut_add_swap: TensorAddSwap<B>,
+ pub pre_mlp_norm: RMSNorm<B>,
+ pub mlp: Box<dyn Mlp<B>>,
+ pub post_mlp_norm: Option<RMSNorm<B>>,
}
impl<B: Backend> LayerExecutables<B> {
@@ -45,7 +44,7 @@ impl<B: Backend> LayerExecutables<B> {
num_groups: usize,
attention_scale: Option<f32>,
decoder_layer_loader: &ParameterTree<B::Context>,
- rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
+ rope: Option<Rc<Rope<B>>>,
) -> Self {
autoreleasepool(|_| {
let ctx = context.as_ref(); // Reference for functions expecting &B::Context
@@ -54,32 +53,28 @@ impl<B: Backend> LayerExecutables<B> {
MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),
MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
};
- let copy_main_to_shortcut: Box<dyn EncodableBlock<B>> = Box::new(
- TensorCopy::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
- )
- .unwrap(),
- );
+ let copy_main_to_shortcut = TensorCopy::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
+ )
+ .unwrap();
- let pre_attention_norm: Box<dyn EncodableBlock<B>> = Box::new(
- RMSNorm::new(
- ctx,
- intermediate_data_type,
- layer_config.pre_attention_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- );
+ let pre_attention_norm = RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ layer_config.pre_attention_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("pre_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel");
let mixer = match &layer_config.mixer_config {
MixerConfig::Attention(attention_config) => {
let rope_block = rope.expect("RoPE encoder missing for attention layer");
- let qkv_projection = linear_block(
+ let qkv_projection = <dyn Linear<B>>::new(
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
@@ -91,7 +86,7 @@ impl<B: Backend> LayerExecutables<B> {
)
.expect("Failed to create qkv projection");
- let qk_norm: Option<Box<dyn EncodableBlock<B>>> =
+ let qk_norm =
if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
match QKNorm::new(
ctx,
@@ -104,14 +99,14 @@ impl<B: Backend> LayerExecutables<B> {
num_groups,
head_dim,
) {
- Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<B>>),
+ Ok(qk_norm) => Some(qk_norm),
Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
}
} else {
None
};
- let out_projection = linear_block(
+ let out_projection = <dyn Linear<B>>::new(
&attention_config.out_projection_config,
attention_config.has_out_biases,
num_heads * head_dim,
@@ -123,18 +118,16 @@ impl<B: Backend> LayerExecutables<B> {
)
.expect("Failed to create out projection");
- let attention = Box::new(
- Attention::new(
- ctx,
- intermediate_data_type,
- layer_index,
- attention_scale,
- attention_config.has_sinks,
- attention_config.is_causal.unwrap_or(true),
- attention_config.sliding_window_size,
- )
- .expect("Failed to create AttentionWrapper kernel"),
- );
+ let attention = Attention::new(
+ ctx,
+ intermediate_data_type,
+ layer_index,
+ attention_scale,
+ attention_config.has_sinks,
+ attention_config.is_causal.unwrap_or(true),
+ attention_config.sliding_window_size,
+ )
+ .expect("Failed to create AttentionWrapper kernel");
MixerExecutables::Attention {
qkv_projection,
@@ -145,7 +138,7 @@ impl<B: Backend> LayerExecutables<B> {
}
},
MixerConfig::Mamba(mamba_config) => {
- let mixer: Box<dyn EncodableBlock<B>> = Box::new(MambaMixer::new(
+ let mixer = MambaMixer::new(
ctx,
layer_type.clone(),
mamba_config.clone(),
@@ -155,65 +148,60 @@ impl<B: Backend> LayerExecutables<B> {
head_dim,
num_groups,
decoder_layer_loader,
- ));
+ );
MixerExecutables::StateSpace {
mixer,
}
},
MixerConfig::ShortConv(short_conv_config) => {
- let mixer: Box<dyn EncodableBlock<B>> = Box::new(ShortConvMixer::new(
+ let mixer = ShortConvMixer::new(
ctx,
layer_type.clone(),
short_conv_config.clone(),
layer_index,
model_dim,
decoder_layer_loader,
- ));
+ );
MixerExecutables::ShortConv {
mixer,
}
},
};
- let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =
- if let Some(norm_config) = &layer_config.post_attention_norm_config {
- Some(Box::new(
- RMSNorm::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- ))
- } else {
- None
- };
-
- let main_shortcut_add_swap: Box<dyn EncodableBlock<B>> = Box::new(
- TensorAddSwap::<B>::new(
- ctx,
- intermediate_data_type,
- vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ let post_attention_norm = if let Some(norm_config) = &layer_config.post_attention_norm_config {
+ Some(
+ RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("post_mixer_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
)
- .unwrap(),
- );
+ } else {
+ None
+ };
- let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(
- RMSNorm::new(
- ctx,
- intermediate_data_type,
- layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- );
+ let main_shortcut_add_swap = TensorAddSwap::<B>::new(
+ ctx,
+ intermediate_data_type,
+ vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
+ )
+ .unwrap();
+
+ let pre_mlp_norm = RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ layer_config.pre_mlp_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel");
- let mlp = mlp_block(
+ let mlp = <dyn Mlp<B>>::new(
&layer_config.mlp_config,
model_dim,
hidden_dim,
@@ -222,22 +210,21 @@ impl<B: Backend> LayerExecutables<B> {
)
.expect("Failed to create mlp block");
- let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =
- if let Some(norm_config) = &layer_config.post_mlp_norm_config {
- Some(Box::new(
- RMSNorm::new(
- ctx,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- ))
- } else {
- None
- };
+ let post_mlp_norm = if let Some(norm_config) = &layer_config.post_mlp_norm_config {
+ Some(
+ RMSNorm::new(
+ ctx,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader.subtree("post_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
+ )
+ } else {
+ None
+ };
Self {
layer_index,
@@ -252,10 +239,8 @@ impl<B: Backend> LayerExecutables<B> {
}
})
}
-}
-impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters,
@@ -269,10 +254,10 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().inputs.clone());
}
- self.copy_main_to_shortcut.encode(state, parameters, command_buffer)?;
+ self.copy_main_to_shortcut.encode(state, command_buffer)?;
// shortcut = input
- self.pre_attention_norm.encode(state, parameters, command_buffer)?;
+ self.pre_attention_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_attention_norm.clone());
@@ -286,13 +271,13 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
attention,
out_projection,
} => {
- qkv_projection.encode(state, parameters, command_buffer)?;
+ qkv_projection.encode(state, command_buffer)?;
if let Some(norm) = qk_norm {
- norm.encode(state, parameters, command_buffer)?;
+ norm.encode(state, command_buffer)?;
}
- rope.encode(state, parameters, command_buffer)?;
+ rope.encode(state, command_buffer)?;
attention.encode(state, parameters, command_buffer)?;
- out_projection.encode(state, parameters, command_buffer)?;
+ out_projection.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
@@ -301,7 +286,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
MixerExecutables::StateSpace {
mixer,
} => {
- mixer.encode(state, parameters, command_buffer)?;
+ mixer.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
@@ -310,7 +295,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
MixerExecutables::ShortConv {
mixer,
} => {
- mixer.encode(state, parameters, command_buffer)?;
+ mixer.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().attention.clone());
@@ -319,7 +304,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
}
if let Some(post_attention_norm) = &self.post_attention_norm {
- post_attention_norm.encode(state, parameters, command_buffer)?;
+ post_attention_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(
@@ -331,7 +316,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
}
//main = attention_result
- self.main_shortcut_add_swap.encode(state, parameters, command_buffer)?;
+ self.main_shortcut_add_swap.encode(state, command_buffer)?;
// shortcut = input + attention_result
// main = input + attention_result
#[cfg(feature = "tracing")]
@@ -339,20 +324,20 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp_inputs.clone());
}
- self.pre_mlp_norm.encode(state, parameters, command_buffer)?;
+ self.pre_mlp_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().pre_mlp_norm.clone());
}
- self.mlp.encode(state, parameters, command_buffer)?;
+ self.mlp.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().mlp.clone());
}
if let Some(post_mlp_norm) = &self.post_mlp_norm {
- post_mlp_norm.encode(state, parameters, command_buffer)?;
+ post_mlp_norm.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
if let Some(ref layer_traces) = layer_traces {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().post_mlp_norm.clone());
@@ -360,7 +345,7 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
}
// main = mlp_result
- self.main_shortcut_add_swap.encode(state, parameters, command_buffer)?;
+ self.main_shortcut_add_swap.encode(state, command_buffer)?;
// shortcut = input + attention_result + mlp_result
// main = input + attention_result + mlp_result
#[cfg(feature = "tracing")]
@@ -368,7 +353,6 @@ impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {
state.encode_copy_array(command_buffer, ArrayId::Main, layer_traces.borrow().outputs.clone());
}
- let _ = parameters;
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rs
index 2ac32fc5d..e709d25b0 100644
--- a/crates/uzu/src/encodable_block/layer/mixer.rs
+++ b/crates/uzu/src/encodable_block/layer/mixer.rs
@@ -2,21 +2,24 @@
use std::rc::Rc;
-use crate::{backends::common::Backend, encodable_block::EncodableBlock};
+use crate::{
+ backends::common::Backend,
+ encodable_block::{Attention, MambaMixer, QKNorm, Rope, ShortConvMixer, linear::Linear},
+};
/// Mixer component - either attention, state space, or short conv.
pub(crate) enum MixerExecutables<B: Backend> {
Attention {
- qkv_projection: Box<dyn EncodableBlock<B>>,
- qk_norm: Option<Box<dyn EncodableBlock<B>>>,
- rope: Rc<Box<dyn EncodableBlock<B>>>,
- attention: Box<dyn EncodableBlock<B>>,
- out_projection: Box<dyn EncodableBlock<B>>,
+ qkv_projection: Box<dyn Linear<B>>,
+ qk_norm: Option<QKNorm<B>>,
+ rope: Rc<Rope<B>>,
+ attention: Attention<B>,
+ out_projection: Box<dyn Linear<B>>,
},
StateSpace {
- mixer: Box<dyn EncodableBlock<B>>,
+ mixer: MambaMixer<B>,
},
ShortConv {
- mixer: Box<dyn EncodableBlock<B>>,
+ mixer: ShortConvMixer<B>,
},
}
diff --git a/crates/uzu/src/encodable_block/layer_norm.rs b/crates/uzu/src/encodable_block/layer_norm.rs
index 0c2ec7e55..301baf9e6 100644
--- a/crates/uzu/src/encodable_block/layer_norm.rs
+++ b/crates/uzu/src/encodable_block/layer_norm.rs
@@ -8,7 +8,6 @@ use std::{
use thiserror::Error;
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -25,7 +24,7 @@ pub enum LayerNormError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
+ ParameterError(ParameterLoaderError<B>),
}
pub struct LayerNorm<B: Backend> {
@@ -45,7 +44,7 @@ impl<B: Backend> LayerNorm<B> {
output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
) -> Result<Self, LayerNormError<B>> {
- let scales = parameter_tree.leaf("scales").map_err(LayerNormError::ParameterError)?;
+ let scales = parameter_tree.leaf_array("scales").map_err(LayerNormError::ParameterError)?;
let accumulation_data_type: DataType = config.accumulation_precision.into();
let scale_data_type: DataType = config.scale_precision.into();
@@ -68,13 +67,10 @@ impl<B: Backend> LayerNorm<B> {
scales_buffer: scales.buffer(),
})
}
-}
-impl<B: Backend> EncodableBlock<B> for LayerNorm<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let input_binding = state.arrays(&[self.input_array_id]);
diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/linear/full_precision.rs
similarity index 93%
rename from crates/uzu/src/encodable_block/full_precision_linear.rs
rename to crates/uzu/src/encodable_block/linear/full_precision.rs
index bde5e27ff..2416b1bb2 100644
--- a/crates/uzu/src/encodable_block/full_precision_linear.rs
+++ b/crates/uzu/src/encodable_block/linear/full_precision.rs
@@ -6,7 +6,7 @@ use std::{
use thiserror::Error;
-use super::{EncodableBlock, EncodingParameters};
+use super::Linear;
use crate::{
DataType,
backends::common::{
@@ -22,7 +22,7 @@ pub enum FullPrecisionLinearError<B: Backend> {
#[error("Matmul error: {0}")]
MatmulError(#[from] MatmulError<B>),
#[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
+ ParameterError(ParameterLoaderError<B>),
#[error("Unsupported data type for full precision linear kernel: {0:?}")]
UnsupportedDataType(DataType),
#[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]
@@ -72,7 +72,7 @@ impl<B: Backend> FullPrecisionLinear<B> {
return Err(FullPrecisionLinearError::UnsupportedDataType(precision));
}
- let weights = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;
+ let weights = parameter_tree.leaf_array("weights").map_err(FullPrecisionLinearError::ParameterError)?;
let weights_shape = weights.shape().to_vec();
if weights_shape != [output_dim, input_dim] {
return Err(FullPrecisionLinearError::InvalidWeightsShape {
@@ -89,7 +89,7 @@ impl<B: Backend> FullPrecisionLinear<B> {
});
}
- let bias_buffer = match parameter_tree.leaf("biases") {
+ let bias_buffer = match parameter_tree.leaf_array("biases") {
Ok(biases) => {
let bias_shape = biases.shape().to_vec();
if bias_shape != [output_dim] {
@@ -125,11 +125,10 @@ impl<B: Backend> FullPrecisionLinear<B> {
}
}
-impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {
+impl<B: Backend> Linear<B> for FullPrecisionLinear<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
diff --git a/crates/uzu/src/encodable_block/linear/mod.rs b/crates/uzu/src/encodable_block/linear/mod.rs
new file mode 100644
index 000000000..ed6f482a5
--- /dev/null
+++ b/crates/uzu/src/encodable_block/linear/mod.rs
@@ -0,0 +1,81 @@
+mod full_precision;
+mod quantized;
+
+pub use full_precision::{FullPrecisionLinear, FullPrecisionLinearError};
+pub use quantized::{QuantizedLinear, QuantizedLinearError};
+use thiserror::Error;
+
+use crate::{
+ backends::common::{Backend, CommandBuffer},
+ config::LinearConfig,
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+pub trait Linear<B: Backend> {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState<B>,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error>;
+}
+
+#[derive(Debug, Error)]
+pub enum LinearBlockError<B: Backend> {
+ #[error("QuantizedLinear error: {0}")]
+ QuantizedLinearError(#[source] QuantizedLinearError<B>),
+ #[error("FullPrecisionLinear error: {0}")]
+ FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[source] ParameterLoaderError<B>),
+ #[error("QLoRA linear layer not supported")]
+ QLoRaNotSupported,
+}
+
+impl<B: Backend> dyn Linear<B> {
+ pub fn new<const N: usize>(
+ config: &LinearConfig,
+ _has_biases: bool,
+ input_dimension: usize,
+ output_dimensions: [usize; N],
+ context: &B::Context,
+ parameter_tree: &ParameterTree<B::Context>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Box<dyn Linear<B>>, LinearBlockError<B>> {
+ let output_dimension_sum: usize = output_dimensions.iter().sum();
+ match config {
+ LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
+ let block = QuantizedLinear::new(
+ context,
+ quantization_config,
+ input_dimension,
+ output_dimension_sum,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )
+ .map_err(LinearBlockError::QuantizedLinearError)?;
+ Ok(Box::new(block))
+ },
+ LinearConfig::FullPrecision {
+ precision,
+ } => {
+ let block = FullPrecisionLinear::new(
+ context,
+ (*precision).into(),
+ input_dimension,
+ output_dimension_sum,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )
+ .map_err(LinearBlockError::FullPrecisionLinearError)?;
+ Ok(Box::new(block))
+ },
+ LinearConfig::QLoRA {
+ ..
+ } => Err(LinearBlockError::QLoRaNotSupported),
+ }
+ }
+}
diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/linear/quantized.rs
similarity index 94%
rename from crates/uzu/src/encodable_block/quantized_linear.rs
rename to crates/uzu/src/encodable_block/linear/quantized.rs
index c2fce2e9c..3f46ab886 100644
--- a/crates/uzu/src/encodable_block/quantized_linear.rs
+++ b/crates/uzu/src/encodable_block/linear/quantized.rs
@@ -6,7 +6,7 @@ use std::{
use thiserror::Error;
-use super::{EncodableBlock, EncodingParameters};
+use super::Linear;
use crate::{
DataType,
backends::common::{
@@ -31,7 +31,7 @@ pub enum QuantizedLinearError<B: Backend> {
#[error("QuantizedMatmul error: {0}")]
QuantizedMatmulError(#[source] QuantizedMatmulError<B>),
#[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
+ ParameterError(ParameterLoaderError<B>),
#[error("Unsupported data type for quantized kernel: {0:?}")]
UnsupportedDataType(DataType),
#[error("Expected weights of type {expected:?}, got {got:?}")]
@@ -114,7 +114,7 @@ impl<B: Backend> QuantizedLinear<B> {
return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));
}
- let weights = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;
+ let weights = parameter_tree.leaf_array("weights").map_err(QuantizedLinearError::ParameterError)?;
let packing_divisor = config.weight_quantization_mode.packing_divisor();
let storage_type = config.weight_quantization_mode.storage_type();
if weights.data_type() != storage_type {
@@ -124,7 +124,7 @@ impl<B: Backend> QuantizedLinear<B> {
});
}
- let scales = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;
+ let scales = parameter_tree.leaf_array("scales").map_err(QuantizedLinearError::ParameterError)?;
if scales.data_type() != kernel_data_type {
return Err(QuantizedLinearError::InvalidScalesDataType {
expected: kernel_data_type,
@@ -137,7 +137,7 @@ impl<B: Backend> QuantizedLinear<B> {
let scales_shape = scales.shape().to_vec();
let weights_transposed = weights_shape[0] == output_dim;
- let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf("deq_biases") {
+ let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf_array("deq_biases") {
Ok(deq_biases) => {
let deq_biases_shape = deq_biases.shape().to_vec();
if !(weights_shape == [output_dim, input_dim / packing_divisor]
@@ -162,7 +162,8 @@ impl<B: Backend> QuantizedLinear<B> {
(QuantizedMatmulType::Mlx, deq_biases.buffer())
},
Err(_) => {
- let zero_points = parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;
+ let zero_points =
+ parameter_tree.leaf_array("zero_points").map_err(QuantizedLinearError::ParameterError)?;
let zero_points_shape = zero_points.shape().to_vec();
let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;
if !(weights_shape == [output_dim, input_dim / packing_divisor]
@@ -189,7 +190,7 @@ impl<B: Backend> QuantizedLinear<B> {
},
};
- let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {
+ let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf_array("biases") {
Ok(biases) => {
let bias_shape = biases.shape().to_vec();
if bias_shape != [output_dim] {
@@ -244,11 +245,10 @@ impl<B: Backend> QuantizedLinear<B> {
}
}
-impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B> {
+impl<B: Backend> Linear<B> for QuantizedLinear<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index 75948b15e..ff2c8b42a 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -16,7 +16,7 @@ use crate::{
},
},
config::{DecoderLayerType, Mamba2Config},
- encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},
+ encodable_block::linear::Linear,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterTree, resolve_subtree},
};
@@ -24,8 +24,8 @@ use crate::{
pub(crate) struct MambaMixer<B: Backend> {
layer_index: usize,
config: Mamba2Config,
- in_projection: Box<dyn EncodableBlock<B>>,
- out_projection: Box<dyn EncodableBlock<B>>,
+ in_projection: Box<dyn Linear<B>>,
+ out_projection: Box<dyn Linear<B>>,
split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,
conv_decode: <B::Kernels as Kernels>::Conv1dDecodeKernel,
conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,
@@ -39,6 +39,17 @@ pub(crate) struct MambaMixer<B: Backend> {
prefill_mode: SSDPrefillMode,
}
+fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
+ match env::var("UZU_SSM_PREFILL_MODE") {
+ Ok(value) => match value.trim().to_ascii_lowercase().as_str() {
+ "seq" | "sequential" | "baseline" => SSDPrefillMode::Sequential,
+ "single" | "singlepass" | "single_pass" => SSDPrefillMode::SinglePass,
+ _ => SSDPrefillMode::SinglePass,
+ },
+ Err(_) => SSDPrefillMode::SinglePass,
+ }
+}
+
impl<B: Backend> MambaMixer<B> {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
@@ -61,7 +72,7 @@ impl<B: Backend> MambaMixer<B> {
let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();
- let in_projection = linear_block(
+ let in_projection = <dyn Linear<B>>::new(
&mamba_config.in_projection_config,
mamba_config.has_in_biases,
model_dim,
@@ -73,7 +84,7 @@ impl<B: Backend> MambaMixer<B> {
)
.expect("Failed to create in-projection kernel");
- let out_projection = linear_block(
+ let out_projection = <dyn Linear<B>>::new(
&mamba_config.out_projection_config,
mamba_config.has_out_biases,
mamba_config.inner_dim(),
@@ -85,14 +96,14 @@ impl<B: Backend> MambaMixer<B> {
)
.expect("Failed to create out-projection kernel");
- let conv_weight = conv_tree.leaf("weights").unwrap();
+ let conv_weight = conv_tree.leaf_array("weights").unwrap();
let conv_bias = if mamba_config.conv_config.has_biases {
- Some(conv_tree.leaf("biases").unwrap())
+ Some(conv_tree.leaf_array("biases").unwrap())
} else {
None
};
- let gate_bias = split_tree.leaf("gate_bias").unwrap();
- let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap();
+ let gate_bias = split_tree.leaf_array("gate_bias").unwrap();
+ let skip_connection_weight = split_tree.leaf_array("skip_connection_weight").unwrap();
let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)
.expect("Failed to create split in-projection kernel");
@@ -136,9 +147,7 @@ impl<B: Backend> MambaMixer<B> {
prefill_mode,
}
}
-}
-impl<B: Backend> MambaMixer<B> {
fn run_split_inproj(
&self,
state: &mut ForwardPassState<B>,
@@ -397,24 +406,10 @@ impl<B: Backend> MambaMixer<B> {
Activation::Gelu => 2,
}
}
-}
-
-fn resolve_prefill_mode_from_env() -> SSDPrefillMode {
- match env::var("UZU_SSM_PREFILL_MODE") {
- Ok(value) => match value.trim().to_ascii_lowercase().as_str() {
- "seq" | "sequential" | "baseline" => SSDPrefillMode::Sequential,
- "single" | "singlepass" | "single_pass" => SSDPrefillMode::SinglePass,
- _ => SSDPrefillMode::SinglePass,
- },
- Err(_) => SSDPrefillMode::SinglePass,
- }
-}
-impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {
- fn encode(
+ pub(crate) fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let active_suffix_length = state.active_suffix_length();
@@ -422,7 +417,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {
return Ok(());
}
- self.in_projection.encode(state, parameters, command_buffer)?;
+ self.in_projection.encode(state, command_buffer)?;
self.run_split_inproj(state, command_buffer, active_suffix_length);
self.run_conv_scan(state, command_buffer, active_suffix_length);
if active_suffix_length == 1 {
@@ -430,7 +425,7 @@ impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {
} else {
self.run_prefill_ssm(state, command_buffer, active_suffix_length);
}
- self.out_projection.encode(state, parameters, command_buffer)?;
+ self.out_projection.encode(state, command_buffer)?;
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp/dense.rs
similarity index 74%
rename from crates/uzu/src/encodable_block/mlp.rs
rename to crates/uzu/src/encodable_block/mlp/dense.rs
index 36c41f65b..1c2e9eac1 100644
--- a/crates/uzu/src/encodable_block/mlp.rs
+++ b/crates/uzu/src/encodable_block/mlp/dense.rs
@@ -2,23 +2,23 @@
use std::ops::{Deref, DerefMut};
+use super::{super::linear::Linear, Mlp};
use crate::{
backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
- encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
};
-pub struct MlpBlock<B: Backend> {
- up: Box<dyn EncodableBlock<B>>,
+pub struct DenseMlp<B: Backend> {
+ up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
- down: Box<dyn EncodableBlock<B>>,
+ down: Box<dyn Linear<B>>,
}
-impl<B: Backend> MlpBlock<B> {
+impl<B: Backend> DenseMlp<B> {
pub fn new(
- up: Box<dyn EncodableBlock<B>>,
+ up: Box<dyn Linear<B>>,
gate: MlpGateActMulEncodable<B>,
- down: Box<dyn EncodableBlock<B>>,
+ down: Box<dyn Linear<B>>,
) -> Self {
Self {
up,
@@ -28,15 +28,14 @@ impl<B: Backend> MlpBlock<B> {
}
}
-impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
+impl<B: Backend> Mlp<B> for DenseMlp<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- params: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
// Up
- self.up.encode(state, params, command_buffer)?;
+ self.up.encode(state, command_buffer)?;
// Gate act+mul (fused_up -> hidden)
let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
@@ -56,7 +55,7 @@ impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {
drop(hidden);
// Down
- self.down.encode(state, params, command_buffer)?;
+ self.down.encode(state, command_buffer)?;
Ok(())
}
}
diff --git a/crates/uzu/src/encodable_block/mlp/mod.rs b/crates/uzu/src/encodable_block/mlp/mod.rs
new file mode 100644
index 000000000..e6d28a089
--- /dev/null
+++ b/crates/uzu/src/encodable_block/mlp/mod.rs
@@ -0,0 +1,85 @@
+mod dense;
+mod moe;
+
+pub use dense::DenseMlp;
+pub use moe::{MoeBlock, MoeBlockError};
+use thiserror::Error;
+
+use super::linear::{Linear, LinearBlockError};
+use crate::{
+ DataType,
+ backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
+ config::MLPConfig,
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+pub trait Mlp<B: Backend> {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState<B>,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<(), B::Error>;
+}
+
+#[derive(Debug, Error)]
+pub enum MlpBlockError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Linear block error: {0}")]
+ LinearBlockError(#[from] LinearBlockError<B>),
+ #[error("MoeBlock error: {0}")]
+ MoeBlockError(#[from] MoeBlockError<B>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[from] ParameterLoaderError<B>),
+}
+
+impl<B: Backend> dyn Mlp<B> {
+ pub fn new(
+ config: &MLPConfig,
+ model_dimension: usize,
+ hidden_dimension: usize,
+ context: &B::Context,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Box<dyn Mlp<B>>, MlpBlockError<B>> {
+ if let MLPConfig::Dense(dense_config) = config {
+ let data_type: DataType = dense_config.linear_config.activation_precision().into();
+
+ let up_projection = <dyn Linear<B>>::new(
+ &dense_config.linear_config,
+ false,
+ model_dimension,
+ [2 * hidden_dimension],
+ context,
+ &parameter_tree.subtree("up_projection")?,
+ ArrayId::Main,
+ ArrayId::MlpFusedUp,
+ )?;
+
+ let gate_activation =
+ MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)
+ .map_err(MlpBlockError::BackendError)?;
+
+ let down_projection = <dyn Linear<B>>::new(
+ &dense_config.linear_config,
+ false,
+ hidden_dimension,
+ [model_dimension],
+ context,
+ &parameter_tree.subtree("down_projection")?,
+ ArrayId::MlpHidden,
+ ArrayId::Main,
+ )?;
+
+ return Ok(Box::new(DenseMlp::new(up_projection, gate_activation, down_projection)));
+ }
+
+ if let MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
+ let mixture_of_experts_block =
+ MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;
+ return Ok(Box::new(mixture_of_experts_block));
+ }
+
+ unreachable!("Unknown MLP config")
+ }
+}
diff --git a/crates/uzu/src/encodable_block/moe_block.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
similarity index 97%
rename from crates/uzu/src/encodable_block/moe_block.rs
rename to crates/uzu/src/encodable_block/mlp/moe.rs
index a5be081ba..7efde157c 100644
--- a/crates/uzu/src/encodable_block/moe_block.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -22,7 +22,7 @@ use crate::{
},
},
},
- encodable_block::{EncodableBlock, EncodingParameters},
+ encodable_block::mlp::Mlp,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -63,7 +63,7 @@ pub enum MoeBlockError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Parameter loader error: {0}")]
- ParameterLoaderError(#[source] ParameterLoaderError),
+ ParameterLoaderError(#[source] ParameterLoaderError<B>),
}
impl<B: Backend> MoeBlock<B> {
@@ -92,8 +92,8 @@ impl<B: Backend> MoeBlock<B> {
LinearConfig::FullPrecision {
..
} => {
- let weights_arr = router_tree.leaf("weights").map_err(MoeBlockError::ParameterLoaderError)?;
- let biases_arr = router_tree.leaf("biases").map_err(MoeBlockError::ParameterLoaderError)?;
+ let weights_arr = router_tree.leaf_array("weights").map_err(MoeBlockError::ParameterLoaderError)?;
+ let biases_arr = router_tree.leaf_array("biases").map_err(MoeBlockError::ParameterLoaderError)?;
RouterBlock {
weights_buf: weights_arr.buffer(),
biases_buf: biases_arr.buffer(),
@@ -128,25 +128,25 @@ impl<B: Backend> MoeBlock<B> {
let w13_arr = experts_tree
.subtree("up_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf("weights")
+ .leaf_array("weights")
.map_err(MoeBlockError::ParameterLoaderError)?;
let w2_arr = experts_tree
.subtree("down_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf("weights")
+ .leaf_array("weights")
.map_err(MoeBlockError::ParameterLoaderError)?;
let up_biases_arr = experts_tree
.subtree("up_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf("biases")
+ .leaf_array("biases")
.map_err(MoeBlockError::ParameterLoaderError)?;
let down_biases_arr = experts_tree
.subtree("down_projection")
.map_err(MoeBlockError::ParameterLoaderError)?
- .leaf("biases")
+ .leaf_array("biases")
.map_err(MoeBlockError::ParameterLoaderError)?;
let shared_weights = SharedMoeWeights {
@@ -188,11 +188,10 @@ impl<B: Backend> MoeBlock<B> {
}
}
-impl<B: Backend> EncodableBlock<B> for MoeBlock<B> {
+impl<B: Backend> Mlp<B> for MoeBlock<B> {
fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let suffix_length = state.active_suffix_length();
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index a973d83c3..b29cb69d1 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -1,61 +1,44 @@
-use crate::{
- backends::common::{Backend, CommandBuffer},
- forward_pass::state::ForwardPassState,
-};
-
mod activation;
mod attention;
mod classifier_layer;
mod decoder;
-mod full_precision_embedding_lookup;
-mod full_precision_embedding_readout;
-mod full_precision_linear;
+mod embedding;
mod layer;
mod layer_norm;
+pub(crate) mod linear;
mod mamba_mixer;
-mod mlp;
-mod moe_block;
+pub(crate) mod mlp;
mod normalization;
mod pooling;
mod prediction_head;
mod qk_norm;
-mod quantized_embedding_lookup;
-mod quantized_embedding_readout;
-mod quantized_linear;
mod rms_norm;
mod rope;
mod sampling;
mod short_conv_mixer;
mod tensor_add_swap;
mod tensor_copy;
-mod transformer_layer;
pub use activation::Activation;
pub use attention::Attention;
pub use classifier_layer::ClassifierLayer;
pub use decoder::Decoder;
-pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};
-pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};
-pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};
+pub use embedding::Embedding;
pub use layer::LayerExecutables;
pub use layer_norm::{LayerNorm, LayerNormError};
+pub use linear::{FullPrecisionLinear, FullPrecisionLinearError, Linear, QuantizedLinear, QuantizedLinearError};
pub(crate) use mamba_mixer::MambaMixer;
-pub use mlp::MlpBlock;
-pub use moe_block::MoeBlock;
+pub use mlp::{DenseMlp, Mlp, MoeBlock, MoeBlockError};
pub use normalization::{Normalization, NormalizationError};
pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use qk_norm::{QKNorm, QKNormError};
-pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};
-pub use quantized_embedding_readout::{QuantizedEmbeddingReadout, QuantizedEmbeddingReadoutError};
-pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};
pub use rms_norm::{RMSNorm, RMSNormError};
pub use rope::Rope;
pub use sampling::Sampling;
pub use short_conv_mixer::ShortConvMixer;
pub use tensor_add_swap::TensorAddSwap;
pub use tensor_copy::TensorCopy;
-pub use transformer_layer::{embed_block, linear_block};
#[derive(Clone)]
pub struct EncodingParameters {
@@ -77,12 +60,3 @@ impl EncodingParameters {
self
}
}
-
-pub trait EncodableBlock<B: Backend> {
- fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error>;
-}
diff --git a/crates/uzu/src/encodable_block/normalization.rs b/crates/uzu/src/encodable_block/normalization.rs
index 308b3fab6..53d5cfd51 100644
--- a/crates/uzu/src/encodable_block/normalization.rs
+++ b/crates/uzu/src/encodable_block/normalization.rs
@@ -2,7 +2,7 @@
use thiserror::Error;
-use super::{EncodableBlock, EncodingParameters, LayerNorm, LayerNormError, RMSNorm, RMSNormError};
+use super::{LayerNorm, LayerNormError, RMSNorm, RMSNormError};
use crate::{
DataType,
backends::common::{Backend, CommandBuffer},
@@ -56,18 +56,15 @@ impl<B: Backend> Normalization<B> {
)?))
}
}
-}
-impl<B: Backend> EncodableBlock<B> for Normalization<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
match self {
- Self::LayerNorm(layer_norm) => layer_norm.encode(state, parameters, command_buffer),
- Self::RMSNorm(rms_norm) => rms_norm.encode(state, parameters, command_buffer),
+ Self::LayerNorm(layer_norm) => layer_norm.encode(state, command_buffer),
+ Self::RMSNorm(rms_norm) => rms_norm.encode(state, command_buffer),
}
}
}
diff --git a/crates/uzu/src/encodable_block/pooling.rs b/crates/uzu/src/encodable_block/pooling.rs
index bf8010fac..b6cfba724 100644
--- a/crates/uzu/src/encodable_block/pooling.rs
+++ b/crates/uzu/src/encodable_block/pooling.rs
@@ -2,7 +2,6 @@
use std::ops::{Deref, DerefMut};
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -58,13 +57,10 @@ impl<B: Backend> Pooling<B> {
model_dim,
})
}
-}
-impl<B: Backend> EncodableBlock<B> for Pooling<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let batch_size = 1;
diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rs
index 46ec73491..62e62ba44 100644
--- a/crates/uzu/src/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/encodable_block/prediction_head.rs
@@ -4,25 +4,25 @@
use crate::forward_pass::state::ArrayId;
use crate::{
backends::common::{Backend, CommandBuffer},
- encodable_block::{EncodableBlock, EncodingParameters},
+ encodable_block::{Activation, Normalization, linear::Linear},
forward_pass::state::ForwardPassState,
};
pub struct ClassifierPredictionHead<B: Backend> {
- dense: Box<dyn EncodableBlock<B>>,
- activation: Box<dyn EncodableBlock<B>>,
- norm: Box<dyn EncodableBlock<B>>,
- readout: Box<dyn EncodableBlock<B>>,
+ dense: Box<dyn Linear<B>>,
+ activation: Activation<B>,
+ norm: Normalization<B>,
+ readout: Box<dyn Linear<B>>,
#[allow(dead_code)]
num_labels: usize,
}
impl<B: Backend> ClassifierPredictionHead<B> {
pub fn new(
- dense: Box<dyn EncodableBlock<B>>,
- activation: Box<dyn EncodableBlock<B>>,
- norm: Box<dyn EncodableBlock<B>>,
- readout: Box<dyn EncodableBlock<B>>,
+ dense: Box<dyn Linear<B>>,
+ activation: Activation<B>,
+ norm: Normalization<B>,
+ readout: Box<dyn Linear<B>>,
num_labels: usize,
) -> Self {
Self {
@@ -33,19 +33,16 @@ impl<B: Backend> ClassifierPredictionHead<B> {
num_labels,
}
}
-}
-impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
- self.dense.encode(state, parameters, command_buffer)?;
- self.activation.encode(state, parameters, command_buffer)?;
- self.norm.encode(state, parameters, command_buffer)?;
- self.readout.encode(state, parameters, command_buffer)?;
+ self.dense.encode(state, command_buffer)?;
+ self.activation.encode(state, command_buffer)?;
+ self.norm.encode(state, command_buffer)?;
+ self.readout.encode(state, command_buffer)?;
#[cfg(feature = "tracing")]
{
diff --git a/crates/uzu/src/encodable_block/qk_norm.rs b/crates/uzu/src/encodable_block/qk_norm.rs
index 0c25f43bf..6c5a7cdab 100644
--- a/crates/uzu/src/encodable_block/qk_norm.rs
+++ b/crates/uzu/src/encodable_block/qk_norm.rs
@@ -8,7 +8,6 @@ use std::{
use thiserror::Error;
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -25,7 +24,7 @@ pub enum QKNormError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
+ ParameterError(ParameterLoaderError<B>),
}
pub struct QKNorm<B: Backend> {
@@ -60,7 +59,7 @@ impl<B: Backend> QKNorm<B> {
// Setup query normalization if configured
if let Some(ref q_config) = query_config {
- let scales = parameter_tree.leaf("query_norm.scales").map_err(QKNormError::ParameterError)?;
+ let scales = parameter_tree.leaf_array("query_norm.scales").map_err(QKNormError::ParameterError)?;
let accumulation_data_type: DataType = q_config.accumulation_precision.into();
let scale_data_type: DataType = q_config.scale_precision.into();
@@ -85,7 +84,7 @@ impl<B: Backend> QKNorm<B> {
// Setup key normalization if configured
if let Some(ref k_config) = key_config {
- let scales = parameter_tree.leaf("key_norm.scales").map_err(QKNormError::ParameterError)?;
+ let scales = parameter_tree.leaf_array("key_norm.scales").map_err(QKNormError::ParameterError)?;
let accumulation_data_type: DataType = k_config.accumulation_precision.into();
let scale_data_type: DataType = k_config.scale_precision.into();
@@ -121,13 +120,10 @@ impl<B: Backend> QKNorm<B> {
head_dim,
})
}
-}
-impl<B: Backend> EncodableBlock<B> for QKNorm<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs b/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
deleted file mode 100644
index ac50416da..000000000
--- a/crates/uzu/src/encodable_block/quantized_embedding_lookup.rs
+++ /dev/null
@@ -1,275 +0,0 @@
-//! Quantized embedding lookup encodable.
-
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
-use thiserror::Error;
-
-use super::{EncodableBlock, EncodingParameters};
-use crate::{
- DataType,
- backends::common::{
- Backend, Buffer, CommandBuffer, Context,
- kernel::{Kernels, QuantizedEmbeddingLookupKernel},
- },
- config::QuantizationMode,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-#[derive(Debug, Error)]
-pub enum QuantizedEmbeddingLookupError<B: Backend> {
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
- #[error("Expected packed weights of type {expected:?}, got {got:?}")]
- InvalidWeightsDataType {
- expected: DataType,
- got: DataType,
- },
- #[error(
- "Embedding lookup weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_packed_dim}]"
- )]
- InvalidWeightsShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_packed_dim: usize,
- },
- #[error(
- "Embedding lookup scales shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"
- )]
- InvalidScalesShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_num_groups: usize,
- },
- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidScalesDataType {
- expected: DataType,
- got: DataType,
- },
- #[error(
- "Embedding lookup biases shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"
- )]
- InvalidBiasesShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_num_groups: usize,
- },
- #[error("Biases dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidBiasesDataType {
- expected: DataType,
- got: DataType,
- },
-}
-
-pub struct QuantizedEmbeddingLookup<B: Backend> {
- kernel: <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel,
- weights_buffer: Rc<RefCell<B::Buffer>>,
- scales_buffer: Rc<RefCell<B::Buffer>>,
- biases_buffer: Rc<RefCell<B::Buffer>>,
- mode: QuantizationMode,
- input_scale: f32,
- vocab_size: u32,
- model_dim: u32,
- group_size: u32,
-}
-
-impl<B: Backend> QuantizedEmbeddingLookup<B> {
- pub fn new_tied(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- input_scale: f32,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, QuantizedEmbeddingLookupError<B>> {
- Self::new_with_names(
- context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- input_scale,
- "weights",
- "scales",
- "biases",
- parameter_tree,
- )
- }
-
- pub fn new_untied_input(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- input_scale: f32,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, QuantizedEmbeddingLookupError<B>> {
- Self::new_with_names(
- context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- input_scale,
- "input_weights",
- "input_scales",
- "input_biases",
- parameter_tree,
- )
- }
-
- fn new_with_names(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- input_scale: f32,
- weights_name: &str,
- scales_name: &str,
- biases_name: &str,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, QuantizedEmbeddingLookupError<B>> {
- let packing_divisor = mode.packing_divisor();
-
- let kernel = <B::Kernels as Kernels>::QuantizedEmbeddingLookupKernel::new(context, data_type)
- .map_err(QuantizedEmbeddingLookupError::BackendError)?;
-
- let weights = parameter_tree.leaf(weights_name).map_err(QuantizedEmbeddingLookupError::ParameterError)?;
-
- if weights.data_type() != mode.storage_type() {
- return Err(QuantizedEmbeddingLookupError::InvalidWeightsDataType {
- expected: mode.storage_type(),
- got: weights.data_type(),
- });
- }
-
- let scales = parameter_tree.leaf(scales_name).map_err(QuantizedEmbeddingLookupError::ParameterError)?;
-
- let num_groups = (model_dim + group_size - 1) / group_size;
- if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingLookupError::InvalidWeightsShape {
- got: weights.shape().to_vec().into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_packed_dim: model_dim / packing_divisor,
- });
- }
-
- if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingLookupError::InvalidScalesShape {
- got: scales.shape().to_vec().into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_num_groups: num_groups,
- });
- }
-
- if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingLookupError::InvalidScalesDataType {
- expected: data_type,
- got: scales.data_type(),
- });
- }
-
- let biases_buffer = match parameter_tree.leaf(biases_name) {
- Ok(biases) => {
- if biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingLookupError::InvalidBiasesShape {
- got: biases.shape().to_vec().into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_num_groups: num_groups,
- });
- }
-
- if biases.data_type() != data_type {
- return Err(QuantizedEmbeddingLookupError::InvalidBiasesDataType {
- expected: data_type,
- got: biases.data_type(),
- });
- }
-
- biases.buffer()
- },
- Err(_) => {
- let element_size = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(QuantizedEmbeddingLookupError::UnsupportedDataType(other));
- },
- };
-
- let size_bytes = vocab_size * num_groups * element_size;
- let buffer = context.create_buffer(size_bytes).map_err(QuantizedEmbeddingLookupError::BackendError)?;
-
- unsafe {
- std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);
- }
-
- Rc::new(RefCell::new(buffer))
- },
- };
-
- Ok(Self {
- kernel,
- weights_buffer: weights.buffer(),
- scales_buffer: scales.buffer(),
- biases_buffer,
- mode,
- input_scale,
- vocab_size: vocab_size as u32,
- model_dim: model_dim as u32,
- group_size: group_size as u32,
- })
- }
-}
-
-impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingLookup<B> {
- fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
- let batch_size = state.active_suffix_length();
- let token_ids_array = arrays[0].borrow_mut();
- let output_array = arrays[1].borrow_mut();
-
- let quant_mode = match self.mode {
- QuantizationMode::UInt4 => 0,
- QuantizationMode::Int8 => 1,
- QuantizationMode::UInt8 => 2,
- };
-
- self.kernel.encode(
- token_ids_array.buffer().borrow().deref(),
- self.weights_buffer.borrow().deref(),
- self.scales_buffer.borrow().deref(),
- self.biases_buffer.borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- batch_size as u32,
- self.vocab_size,
- self.model_dim,
- self.group_size,
- self.input_scale,
- quant_mode,
- command_buffer,
- );
- Ok(())
- }
-}
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
deleted file mode 100644
index 791237300..000000000
--- a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
+++ /dev/null
@@ -1,270 +0,0 @@
-//! Quantized embedding readout encodable.
-
-use std::{
- cell::RefCell,
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
-use thiserror::Error;
-
-use super::{EncodableBlock, EncodingParameters};
-use crate::{
- DataType,
- backends::common::{
- Backend, Buffer, CommandBuffer, Context,
- kernel::quant_matmul::{
- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
- QuantizedMatmulKernelEncodable, QuantizedMatmulType,
- },
- },
- config::QuantizationMode,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-#[derive(Debug, Error)]
-pub enum QuantizedEmbeddingReadoutError<B: Backend> {
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("Matmul error: {0}")]
- MatmulError(#[source] QuantizedMatmulError<B>),
- #[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
- #[error(
- "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_packed_dim}]"
- )]
- InvalidWeightsShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_packed_dim: usize,
- },
- #[error(
- "Embedding readout scales shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"
- )]
- InvalidScalesShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_num_groups: usize,
- },
- #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidScalesDataType {
- expected: DataType,
- got: DataType,
- },
- #[error(
- "Embedding readout deq_biases shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"
- )]
- InvalidBiasesShape {
- got: Box<[usize]>,
- expected_vocab_size: usize,
- expected_num_groups: usize,
- },
- #[error("Biases dtype mismatch: got {got:?}, expected {expected:?}")]
- InvalidBiasesDataType {
- expected: DataType,
- got: DataType,
- },
-}
-
-pub struct QuantizedEmbeddingReadout<B: Backend> {
- kernel: QuantizedMatmulKernelEncodable<B>,
- weights_buffer: Rc<RefCell<B::Buffer>>,
- scales_buffer: Rc<RefCell<B::Buffer>>,
- biases_buffer: Rc<RefCell<B::Buffer>>,
- vocab_size: usize,
- model_dim: usize,
-}
-
-impl<B: Backend> QuantizedEmbeddingReadout<B> {
- pub fn new_tied(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {
- Self::new_with_names(
- context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- "weights",
- "scales",
- "biases",
- parameter_tree,
- )
- }
-
- pub fn new_untied_output(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {
- Self::new_with_names(
- context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- "output_weights",
- "output_scales",
- "output_biases",
- parameter_tree,
- )
- }
-
- fn new_with_names(
- context: &B::Context,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- weights_name: &str,
- scales_name: &str,
- biases_name: &str,
- parameter_tree: &ParameterTree<B::Context>,
- ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {
- let weights = parameter_tree.leaf(weights_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;
- let scales = parameter_tree.leaf(scales_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;
-
- let num_groups = (model_dim + group_size - 1) / group_size;
- let packing_divisor = mode.packing_divisor();
- let weights_transposed = weights.shape()[0] == vocab_size;
-
- if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingReadoutError::InvalidWeightsShape {
- got: weights.shape().to_vec().into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_packed_dim: model_dim / packing_divisor,
- });
- }
-
- if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingReadoutError::InvalidScalesShape {
- got: scales.shape().to_vec().into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_num_groups: num_groups,
- });
- }
-
- if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingReadoutError::InvalidScalesDataType {
- expected: data_type,
- got: scales.data_type(),
- });
- }
-
- let biases_buffer = match parameter_tree.leaf(biases_name) {
- Ok(deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingReadoutError::InvalidBiasesShape {
- got: deq_biases.shape().to_vec().into_boxed_slice(),
- expected_vocab_size: vocab_size,
- expected_num_groups: num_groups,
- });
- }
- if deq_biases.data_type() != data_type {
- return Err(QuantizedEmbeddingReadoutError::InvalidBiasesDataType {
- expected: data_type,
- got: deq_biases.data_type(),
- });
- }
- deq_biases.buffer()
- },
- Err(_) => {
- let element_size = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(QuantizedEmbeddingReadoutError::UnsupportedDataType(other));
- },
- };
- let size_bytes = vocab_size * num_groups * element_size;
- let buffer = context.create_buffer(size_bytes).map_err(QuantizedEmbeddingReadoutError::BackendError)?;
-
- unsafe {
- std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);
- }
-
- Rc::new(RefCell::new(buffer))
- },
- };
-
- let kernel = QuantizedMatmulKernelEncodable::new(
- context,
- QuantizedMatmulConfiguration {
- data_type,
- group_size,
- input_dim: model_dim,
- output_dim: vocab_size,
- mode,
- quantization_type: QuantizedMatmulType::Mlx,
- weights_transposed,
- },
- )
- .map_err(QuantizedEmbeddingReadoutError::MatmulError)?;
-
- Ok(Self {
- kernel,
- weights_buffer: weights.buffer(),
- scales_buffer: scales.buffer(),
- biases_buffer,
- vocab_size,
- model_dim,
- })
- }
-}
-
-impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {
- fn encode(
- &self,
- state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
- command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
- ) -> Result<(), B::Error> {
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.sampling_length();
- if batch_size == 0 {
- return Ok(());
- }
-
- let sampling_start = state.sampling_start();
- let input_array = arrays[0].borrow_mut();
- let output_array = arrays[1].borrow_mut();
-
- let element_size = input_array.data_type().size_in_bytes();
- let a_offset = sampling_start * self.model_dim * element_size;
-
- self.kernel
- .encode(
- command_buffer,
- QuantizedMatmulArguments {
- a_buffer: input_array.buffer().borrow().deref(),
- a_offset,
- b_buffer: self.weights_buffer.borrow().deref(),
- scales_buffer: self.scales_buffer.borrow().deref(),
- zero_points_or_biases_buffer: self.biases_buffer.borrow().deref(),
- output_buffer: output_array.buffer().borrow_mut().deref_mut(),
- batch: batch_size,
- input_dim: self.model_dim,
- output_dim: self.vocab_size,
- quantization_type: QuantizedMatmulType::Mlx,
- },
- )
- .expect("Failed to encode quantized embedding readout kernel");
- Ok(())
- }
-}
diff --git a/crates/uzu/src/encodable_block/rms_norm.rs b/crates/uzu/src/encodable_block/rms_norm.rs
index 13a1c7483..30ecb88f1 100644
--- a/crates/uzu/src/encodable_block/rms_norm.rs
+++ b/crates/uzu/src/encodable_block/rms_norm.rs
@@ -8,7 +8,6 @@ use std::{
use thiserror::Error;
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -25,7 +24,7 @@ pub enum RMSNormError<B: Backend> {
#[error("Backend error: {0}")]
BackendError(#[source] B::Error),
#[error("Parameter loading error: {0}")]
- ParameterError(ParameterLoaderError),
+ ParameterError(ParameterLoaderError<B>),
}
pub struct RMSNorm<B: Backend> {
@@ -46,7 +45,7 @@ impl<B: Backend> RMSNorm<B> {
output_array_id: ArrayId,
parameter_tree: &ParameterTree<B::Context>,
) -> Result<Self, RMSNormError<B>> {
- let scales = parameter_tree.leaf("scales").map_err(RMSNormError::ParameterError)?;
+ let scales = parameter_tree.leaf_array("scales").map_err(RMSNormError::ParameterError)?;
let accumulation_data_type: DataType = config.accumulation_precision.into();
let scale_data_type: DataType = config.scale_precision.into();
@@ -82,13 +81,10 @@ impl<B: Backend> RMSNorm<B> {
self.use_sampling_range = true;
self
}
-}
-impl<B: Backend> EncodableBlock<B> for RMSNorm<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let input_binding = state.arrays(&[self.input_array_id]);
diff --git a/crates/uzu/src/encodable_block/rope.rs b/crates/uzu/src/encodable_block/rope.rs
index 66ca919f2..4e3aec68c 100644
--- a/crates/uzu/src/encodable_block/rope.rs
+++ b/crates/uzu/src/encodable_block/rope.rs
@@ -2,7 +2,6 @@
use std::ops::{Deref, DerefMut};
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -28,13 +27,10 @@ impl<B: Backend> Rope<B> {
rope_type,
})
}
-}
-impl<B: Backend> EncodableBlock<B> for Rope<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
diff --git a/crates/uzu/src/encodable_block/sampling.rs b/crates/uzu/src/encodable_block/sampling.rs
index 2b4901f0b..f63ff47a5 100644
--- a/crates/uzu/src/encodable_block/sampling.rs
+++ b/crates/uzu/src/encodable_block/sampling.rs
@@ -2,7 +2,6 @@
use std::ops::{Deref, DerefMut};
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -42,13 +41,10 @@ impl<B: Backend> Sampling<B> {
kernel,
})
}
-}
-impl<B: Backend> EncodableBlock<B> for Sampling<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
assert!(state.sampling_output().is_some(), "Sampling output buffer must be pre-allocated");
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index 75534176f..98406a2e7 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -8,7 +8,7 @@ use crate::{
kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
},
config::{DecoderLayerType, ShortConvConfig},
- encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},
+ encodable_block::linear::Linear,
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterTree, resolve_subtree},
};
@@ -17,8 +17,8 @@ pub struct ShortConvMixer<B: Backend> {
layer_index: usize,
config: ShortConvConfig,
model_dim: usize,
- in_projection: Box<dyn EncodableBlock<B>>,
- out_projection: Box<dyn EncodableBlock<B>>,
+ in_projection: Box<dyn Linear<B>>,
+ out_projection: Box<dyn Linear<B>>,
short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,
short_conv_prefill: <B::Kernels as Kernels>::ShortConvPrefillKernel,
short_conv_decode: <B::Kernels as Kernels>::ShortConvDecodeKernel,
@@ -45,7 +45,7 @@ impl<B: Backend> ShortConvMixer<B> {
let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();
- let in_projection = linear_block(
+ let in_projection = <dyn Linear<B>>::new(
&short_conv_config.in_projection_config,
false,
model_dim,
@@ -57,7 +57,7 @@ impl<B: Backend> ShortConvMixer<B> {
)
.expect("Failed to create in-projection kernel");
- let out_projection = linear_block(
+ let out_projection = <dyn Linear<B>>::new(
&short_conv_config.out_projection_config,
false,
model_dim,
@@ -69,9 +69,9 @@ impl<B: Backend> ShortConvMixer<B> {
)
.expect("Failed to create out-projection kernel");
- let conv_weight = conv_tree.leaf("weights").unwrap().clone();
+ let conv_weight = conv_tree.leaf_array("weights").unwrap().clone();
let conv_bias = if short_conv_config.conv_config.has_biases {
- Some(conv_tree.leaf("biases").unwrap().clone())
+ Some(conv_tree.leaf_array("biases").unwrap().clone())
} else {
None
};
@@ -100,9 +100,7 @@ impl<B: Backend> ShortConvMixer<B> {
conv_bias,
}
}
-}
-impl<B: Backend> ShortConvMixer<B> {
fn clear_suffix_state_valid_range(
&self,
state: &ForwardPassState<B>,
@@ -322,13 +320,10 @@ impl<B: Backend> ShortConvMixer<B> {
compute,
)
}
-}
-impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let active_suffix_length = state.active_suffix_length();
@@ -336,11 +331,11 @@ impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {
return Ok(());
}
- self.in_projection.encode(state, parameters, command_buffer)?;
+ self.in_projection.encode(state, command_buffer)?;
self.run_conv(state, command_buffer, active_suffix_length);
- self.out_projection.encode(state, parameters, command_buffer)?;
+ self.out_projection.encode(state, command_buffer)?;
Ok(())
}
diff --git a/crates/uzu/src/encodable_block/tensor_add_swap.rs b/crates/uzu/src/encodable_block/tensor_add_swap.rs
index 627208b12..ea0b2b9d1 100644
--- a/crates/uzu/src/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/encodable_block/tensor_add_swap.rs
@@ -2,7 +2,6 @@
use std::ops::DerefMut;
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -29,13 +28,10 @@ impl<B: Backend> TensorAddSwap<B> {
argument_arrays,
})
}
-}
-impl<B: Backend> EncodableBlock<B> for TensorAddSwap<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let arrays = state.arrays(&self.argument_arrays);
diff --git a/crates/uzu/src/encodable_block/tensor_copy.rs b/crates/uzu/src/encodable_block/tensor_copy.rs
index d82361ead..74af8ac92 100644
--- a/crates/uzu/src/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/encodable_block/tensor_copy.rs
@@ -2,7 +2,6 @@
use std::ops::{Deref, DerefMut};
-use super::{EncodableBlock, EncodingParameters};
use crate::{
DataType,
backends::common::{
@@ -29,13 +28,10 @@ impl<B: Backend> TensorCopy<B> {
argument_arrays,
})
}
-}
-impl<B: Backend> EncodableBlock<B> for TensorCopy<B> {
- fn encode(
+ pub fn encode(
&self,
state: &mut ForwardPassState<B>,
- _parameters: &EncodingParameters,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<(), B::Error> {
let arrays = state.arrays(&self.argument_arrays);
diff --git a/crates/uzu/src/encodable_block/transformer_layer.rs b/crates/uzu/src/encodable_block/transformer_layer.rs
deleted file mode 100644
index 736fe8fb3..000000000
--- a/crates/uzu/src/encodable_block/transformer_layer.rs
+++ /dev/null
@@ -1,405 +0,0 @@
-use thiserror::Error;
-
-use super::{
- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, QuantizedEmbeddingLookup,
- QuantizedEmbeddingReadout, QuantizedLinear,
-};
-use crate::{
- DataType,
- backends::common::{Backend, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
- config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
- encodable_block::{
- EncodableBlock, FullPrecisionLinearError, MlpBlock, MoeBlock, QuantizedLinearError, moe_block::MoeBlockError,
- },
- forward_pass::state::ArrayId,
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-#[derive(Debug, Error)]
-pub enum LayerError<B: Backend> {
- #[error("Backend error: {0}")]
- BackendError(#[source] B::Error),
- #[error("QuantizedLinear error: {0}")]
- QuantizedLinearError(#[source] QuantizedLinearError<B>),
- #[error("FullPrecisionLinear error: {0}")]
- FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),
- #[error("MoeBlock error: {0}")]
- MoeBlockError(#[source] MoeBlockError<B>),
- #[error("Parameter loader error: {0}")]
- ParameterLoaderError(#[source] ParameterLoaderError),
- #[error("QLoRA linear layer not supported")]
- QLoRaNotSupported,
-}
-
-pub fn linear_block<const N: usize, B: Backend>(
- config: &LinearConfig,
- _has_biases: bool,
- input_dimension: usize,
- output_dimensions: [usize; N],
- context: &B::Context,
- parameter_tree: &ParameterTree<B::Context>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {
- let output_dimension_sum: usize = output_dimensions.iter().sum();
- match config {
- LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
- let block = QuantizedLinear::new(
- context,
- quantization_config,
- input_dimension,
- output_dimension_sum,
- parameter_tree,
- input_array_id,
- output_array_id,
- )
- .map_err(LayerError::QuantizedLinearError)?;
- Ok(Box::new(block))
- },
- LinearConfig::FullPrecision {
- precision,
- } => {
- let block = FullPrecisionLinear::new(
- context,
- (*precision).into(),
- input_dimension,
- output_dimension_sum,
- parameter_tree,
- input_array_id,
- output_array_id,
- )
- .map_err(LayerError::FullPrecisionLinearError)?;
- Ok(Box::new(block))
- },
- LinearConfig::QLoRA {
- ..
- } => Err(LayerError::QLoRaNotSupported),
- }
-}
-
-/// Creates an MLP block using the unfused implementation (separate up, activation, down)
-pub fn mlp_block<B: Backend>(
- config: &MLPConfig,
- model_dimension: usize,
- hidden_dimension: usize,
- context: &B::Context,
- parameter_tree: &ParameterTree<B::Context>,
-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {
- if let crate::config::MLPConfig::Dense(dense_config) = config {
- let data_type: DataType = dense_config.linear_config.activation_precision().into();
-
- // Up projection (outputs 2*hidden_dimension for gate and up)
- let up_projection = linear_block(
- &dense_config.linear_config,
- false,
- model_dimension,
- [2 * hidden_dimension],
- context,
- &parameter_tree.subtree("up_projection").map_err(LayerError::ParameterLoaderError)?,
- ArrayId::Main,
- ArrayId::MlpFusedUp,
- )?;
-
- // Gate activation + multiply
- let gate_activation =
- MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)
- .map_err(LayerError::BackendError)?;
-
- // Down projection
- let down_projection = linear_block(
- &dense_config.linear_config,
- false,
- hidden_dimension,
- [model_dimension],
- context,
- &parameter_tree.subtree("down_projection").map_err(LayerError::ParameterLoaderError)?,
- ArrayId::MlpHidden,
- ArrayId::Main,
- )?;
-
- return Ok(Box::new(MlpBlock::new(up_projection, gate_activation, down_projection)));
- }
-
- if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
- let mixture_of_experts_block =
- MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)
- .map_err(LayerError::MoeBlockError)?;
- return Ok(Box::new(mixture_of_experts_block));
- }
-
- unreachable!("Unknown MLP config")
-}
-
-pub fn embed_block<B: Backend>(
- config: &DecoderConfig,
- context: &B::Context,
- parameter_tree: &ParameterTree<B::Context>,
-) -> Box<dyn EncodableBlock<B>> {
- match &config.embedding_config {
- EmbeddingConfig::Tied {
- common,
- precision,
- } => {
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = FullPrecisionEmbeddingLookup::new(
- context,
- (*precision).into(),
- config.vocab_size,
- config.model_dim,
- common.input_scale,
- &embeddings_tree,
- )
- .expect("Failed to create full precision embedding lookup");
-
- Box::new(block)
- },
- EmbeddingConfig::Untied {
- common,
- precision,
- } => {
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = FullPrecisionEmbeddingLookup::new(
- context,
- (*precision).into(),
- config.vocab_size,
- config.model_dim,
- common.input_scale,
- &embeddings_tree,
- )
- .expect("Failed to create full precision embedding lookup");
-
- Box::new(block)
- },
- EmbeddingConfig::MLXSemiQuantizedUntied {
- common,
- activation_precision,
- ..
- } => {
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = FullPrecisionEmbeddingLookup::new(
- context,
- (*activation_precision).into(),
- config.vocab_size,
- config.model_dim,
- common.input_scale,
- &embeddings_tree,
- )
- .expect("Failed to create full precision embedding lookup");
-
- Box::new(block)
- },
- EmbeddingConfig::MLXQuantizedUntied {
- group_size,
- embedding_quantization_mode,
- activation_precision,
- ..
- } => {
- let data_type: DataType = (*activation_precision).into();
- let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);
-
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = QuantizedEmbeddingLookup::new_untied_input(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- *group_size,
- *embedding_quantization_mode,
- input_scale,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding lookup kernel");
-
- Box::new(block)
- },
- EmbeddingConfig::QuantizedTied {
- embedding_quantization_mode,
- activation_precision,
- ..
- } => {
- let data_type: DataType = (*activation_precision).into();
-
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim
- let group_size = config.model_dim;
- let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);
-
- let block = QuantizedEmbeddingLookup::new_tied(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- group_size,
- *embedding_quantization_mode,
- input_scale,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding lookup kernel");
-
- Box::new(block)
- },
- EmbeddingConfig::MLXQuantizedTied {
- group_size,
- embedding_quantization_mode,
- ..
- } => {
- let data_type: DataType = config.output_norm_config.scale_precision.into();
- let input_scale = config.embedding_config.common().input_scale.unwrap_or(1.0);
-
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = QuantizedEmbeddingLookup::new_tied(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- *group_size,
- *embedding_quantization_mode,
- input_scale,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding lookup kernel");
-
- Box::new(block)
- },
- }
-}
-
-pub fn readout_block<B: Backend>(
- config: &DecoderConfig,
- context: &B::Context,
- parameter_tree: &ParameterTree<B::Context>,
-) -> Box<dyn EncodableBlock<B>> {
- match &config.embedding_config {
- EmbeddingConfig::Tied {
- precision,
- ..
- } => {
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = FullPrecisionEmbeddingReadout::new(
- context,
- (*precision).into(),
- config.vocab_size,
- config.model_dim,
- &embeddings_tree,
- )
- .expect("Failed to create full precision embedding readout");
-
- Box::new(block)
- },
- EmbeddingConfig::Untied {
- precision,
- ..
- } => {
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = FullPrecisionEmbeddingReadout::new(
- context,
- (*precision).into(),
- config.vocab_size,
- config.model_dim,
- &embeddings_tree,
- )
- .expect("Failed to create full precision embedding readout");
-
- Box::new(block)
- },
- EmbeddingConfig::MLXSemiQuantizedUntied {
- group_size,
- activation_precision,
- embedding_quantization_mode,
- ..
- } => {
- let data_type: DataType = (*activation_precision).into();
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = QuantizedEmbeddingReadout::new_untied_output(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- *group_size,
- *embedding_quantization_mode,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding readout kernel");
-
- Box::new(block)
- },
- EmbeddingConfig::QuantizedTied {
- embedding_quantization_mode,
- activation_precision,
- ..
- } => {
- let data_type: DataType = (*activation_precision).into();
-
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim
- let group_size = config.model_dim;
-
- let block = QuantizedEmbeddingReadout::new_tied(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- group_size,
- *embedding_quantization_mode,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding readout kernel");
-
- Box::new(block)
- },
- EmbeddingConfig::MLXQuantizedTied {
- group_size,
- embedding_quantization_mode,
- ..
- } => {
- let data_type: DataType = config.output_norm_config.scale_precision.into();
-
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = QuantizedEmbeddingReadout::new_tied(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- *group_size,
- *embedding_quantization_mode,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding readout kernel");
-
- Box::new(block)
- },
- EmbeddingConfig::MLXQuantizedUntied {
- group_size,
- embedding_quantization_mode,
- ..
- } => {
- let data_type: DataType = config.output_norm_config.scale_precision.into();
-
- let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");
-
- let block = QuantizedEmbeddingReadout::new_untied_output(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- *group_size,
- *embedding_quantization_mode,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding readout kernel");
-
- Box::new(block)
- },
- }
-}
diff --git a/crates/uzu/src/forward_pass/state/rope_buffers.rs b/crates/uzu/src/forward_pass/state/rope_buffers.rs
index d21a65db2..628e63e12 100644
--- a/crates/uzu/src/forward_pass/state/rope_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/rope_buffers.rs
@@ -46,10 +46,10 @@ impl<B: Backend> RopeBuffers<B> {
return;
};
- let cosines_view = rope_tree.leaf("cosines").unwrap();
+ let cosines_view = rope_tree.leaf_array("cosines").unwrap();
self.cosines.borrow_mut().copy_from_array(&cosines_view);
- let sines_view = rope_tree.leaf("sines").unwrap();
+ let sines_view = rope_tree.leaf_array("sines").unwrap();
self.sines.borrow_mut().copy_from_array(&sines_view);
}
}
diff --git a/crates/uzu/src/forward_pass/state/shared_buffers.rs b/crates/uzu/src/forward_pass/state/shared_buffers.rs
index 2d2e02980..2cfa6b6e6 100644
--- a/crates/uzu/src/forward_pass/state/shared_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/shared_buffers.rs
@@ -65,7 +65,7 @@ impl<B: Backend> SharedBuffers<B> {
for (layer_idx, sink_cell) in sinks_vec.iter_mut().enumerate() {
let layer_tree = transformer_tree.subtree(&format!("layers.{}", layer_idx)).unwrap();
let attn_tree = layer_tree.subtree("mixer").unwrap();
- let sinks_arr = attn_tree.leaf("sinks").unwrap();
+ let sinks_arr = attn_tree.leaf_array("sinks").unwrap();
let mut dst = sink_cell.borrow_mut();
let dst_slice = dst.as_slice_mut::<f32>();
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index dcc57048d..7202de709 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -21,7 +21,7 @@ use crate::{
CommandBufferPending, Context,
kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},
},
- encodable_block::{EncodableBlock, EncodingParameters},
+ encodable_block::EncodingParameters,
forward_pass::{
cache_layers::{CacheLayer, CacheLayersSlice},
kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},
@@ -536,7 +536,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// Encode sampling
self.context
.gpu_sampler
- .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)
+ .encode(&mut state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
// Copy sampled token: sampling_output → token_ids (for next pass)
@@ -885,7 +885,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
if sample {
self.context
.gpu_sampler
- .encode(state, parameters, &mut command_buffer)
+ .encode(state, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rs
index be1380602..529a8361e 100644
--- a/crates/uzu/src/parameters/loader.rs
+++ b/crates/uzu/src/parameters/loader.rs
@@ -10,7 +10,7 @@ use super::safetensors_metadata::{HashMetadata as STMetadata, HeaderLoadingError
use crate::{
DataType,
array::{Array, ArrayContextExt},
- backends::common::Context,
+ backends::common::{Backend, Buffer, Context},
};
pub struct ParameterMetadata {
@@ -43,7 +43,7 @@ fn st_metadata_into_index(
}
#[derive(Debug, Error)]
-pub enum ParameterLoaderError {
+pub enum ParameterLoaderError<B: Backend> {
#[error("Array with key \"{0}\" not found.")]
KeyNotFound(String),
#[error("Couldn't find any arrays with prefix \"{0}\".")]
@@ -58,6 +58,8 @@ pub enum ParameterLoaderError {
expected_size: usize,
actual_size: usize,
},
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
#[error("Failed to read data")]
ArrayLoadingError(#[from] std::io::Error),
}
@@ -92,10 +94,21 @@ where
self.index.keys()
}
+ pub fn get_leaf<'leaf>(
+ &'leaf self,
+ key: &str,
+ ) -> Result<ParameterLeaf<'file, 'context, 'leaf, C>, ParameterLoaderError<C::Backend>> {
+ Ok(ParameterLeaf {
+ key: key.to_string(),
+ metadata: self.index.get(key).ok_or_else(|| ParameterLoaderError::KeyNotFound(key.to_string()))?,
+ loader: self,
+ })
+ }
+
pub fn get(
&self,
key: &str,
- ) -> Result<Array<C::Backend>, ParameterLoaderError> {
+ ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {
let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
let (offset, size) = (metadata_entry.offset, metadata_entry.size);
let array_key = key.replace(".", "_");
@@ -120,7 +133,7 @@ where
buf: &mut [u8],
shape: &mut Box<[usize]>,
data_type: &mut DataType,
- ) -> Result<(), ParameterLoaderError> {
+ ) -> Result<(), ParameterLoaderError<C::Backend>> {
let metadata_entry = self.index.get(key).ok_or(ParameterLoaderError::KeyNotFound(key.to_string()))?;
self.file.read_exact_at(buf, metadata_entry.offset as u64)?;
*shape = metadata_entry.shape.to_owned();
@@ -136,6 +149,37 @@ where
}
}
+pub struct ParameterLeaf<'file, 'context, 'leaf, C: Context> {
+ key: String,
+ metadata: &'leaf ParameterMetadata,
+ loader: &'leaf ParameterLoader<'context, 'file, C>,
+}
+
+impl<'file, 'context, 'leaf, C: Context> ParameterLeaf<'file, 'context, 'leaf, C> {
+ pub fn shape(&self) -> &[usize] {
+ &self.metadata.shape
+ }
+
+ pub fn data_type(&self) -> DataType {
+ self.metadata.data_type
+ }
+
+ pub fn size(&self) -> usize {
+ self.metadata.size
+ }
+
+ pub fn read_buffer(&self) -> Result<<C::Backend as Backend>::Buffer, ParameterLoaderError<C::Backend>> {
+ let mut buffer =
+ self.loader.context.create_buffer(self.metadata.size).map_err(ParameterLoaderError::BackendError)?;
+ buffer.set_label(Some(&format!("parameter_loader_{}", self.key.replace(".", "_"))));
+ self.loader.file.read_exact_at(
+ unsafe { std::slice::from_raw_parts_mut(buffer.cpu_ptr().as_ptr() as *mut u8, self.metadata.size) },
+ self.metadata.offset as u64,
+ )?;
+ Ok(buffer)
+ }
+}
+
pub struct ParameterTree<'loader, C: Context> {
loader: &'loader ParameterLoader<'loader, 'loader, C>,
prefix: Option<String>,
@@ -163,7 +207,7 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
pub fn subtree(
&self,
name: &str,
- ) -> Result<Self, ParameterLoaderError> {
+ ) -> Result<Self, ParameterLoaderError<C::Backend>> {
let new_prefix = self.join_prefix(name);
let num_suffixes = self.loader.keys().filter_map(|suffix| suffix.strip_prefix(&new_prefix)).count();
if num_suffixes > 0 {
@@ -176,20 +220,27 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {
}
}
- pub fn leaf(
+ pub fn leaf_array(
&self,
name: &str,
- ) -> Result<Array<C::Backend>, ParameterLoaderError> {
+ ) -> Result<Array<C::Backend>, ParameterLoaderError<C::Backend>> {
self.loader.get(&self.join_prefix(name))
}
+ pub fn leaf<'leaf>(
+ &'leaf self,
+ name: &str,
+ ) -> Result<ParameterLeaf<'loader, 'loader, 'leaf, C>, ParameterLoaderError<C::Backend>> {
+ self.loader.get_leaf(&self.join_prefix(name))
+ }
+
pub fn read_extract_at(
&self,
name: &str,
buf: &mut [u8],
shape: &mut Box<[usize]>,
data_type: &mut DataType,
- ) -> Result<(), ParameterLoaderError> {
+ ) -> Result<(), ParameterLoaderError<C::Backend>> {
self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)
}
}
diff --git a/crates/uzu/src/parameters/mod.rs b/crates/uzu/src/parameters/mod.rs
index 903847308..c5ebfd4ac 100644
--- a/crates/uzu/src/parameters/mod.rs
+++ b/crates/uzu/src/parameters/mod.rs
@@ -1,5 +1,5 @@
mod loader;
-pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};
+pub use loader::{ParameterLeaf, ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};
mod safetensors_metadata;
// Re-export the safetensors header reader so other modules (e.g. decoder
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index 027b25f73..848bcb579 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -24,7 +24,7 @@ use crate::{
},
classifier::Classifier,
config::ModelMetadata,
- encodable_block::{EncodableBlock, EncodingParameters, Sampling},
+ encodable_block::{EncodingParameters, Sampling},
forward_pass::{
cache_layers::CacheLayers,
scratch_buffers::ScratchBuffers,
@@ -297,7 +297,7 @@ impl<B: Backend> TraceValidator<B> {
for index in transformer_layers {
let arrays = state.arrays(&[ArrayId::Keys(index), ArrayId::Values(index)]);
- if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.keys", index)) {
+ if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {
let keys = arrays[0].borrow();
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.keys", index),
@@ -305,7 +305,7 @@ impl<B: Backend> TraceValidator<B> {
});
}
- if let Ok(expected) = traces_view.leaf(&format!("updated_kv_cache.{}.values", index)) {
+ if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {
let values = arrays[1].borrow();
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.values", index),
@@ -329,7 +329,7 @@ impl<B: Backend> TraceValidator<B> {
format!("updated_state.{}.conv_state", index),
format!("activation_trace.layer_results.{}.updated_state.conv_state", index),
] {
- if let Ok(expected) = traces_view.leaf(&path) {
+ if let Ok(expected) = traces_view.leaf_array(&path) {
results.push(TracerValidationResult {
name: path,
metrics: Self::validate_array(
@@ -346,7 +346,7 @@ impl<B: Backend> TraceValidator<B> {
format!("updated_state.{}.ssm_state", index),
format!("activation_trace.layer_results.{}.updated_state.ssm_state", index),
] {
- if let Ok(expected) = traces_view.leaf(&path) {
+ if let Ok(expected) = traces_view.leaf_array(&path) {
results.push(TracerValidationResult {
name: path,
metrics: Self::validate_array(data_type, &expected, &ssm_state, None),
@@ -356,7 +356,7 @@ impl<B: Backend> TraceValidator<B> {
}
// LLM-specific: Token comparison
- let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf("logits") {
+ let tokens_violation_indices = if let Ok(expected_logits) = traces_view.leaf_array("logits") {
let expected_tokens = Self::get_tokens_from_logits(&expected_logits);
let produced_tokens = Self::get_tokens_from_logits(&*traces.borrow().logits.borrow());
expected_tokens
@@ -396,8 +396,8 @@ impl<B: Backend> TraceValidator<B> {
ParameterLoader::new(&traces_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
let traces_view = traces_loader.tree();
- let has_token_ids = traces_view.leaf("activation_trace.token_ids").is_ok();
- let has_token_positions = traces_view.leaf("activation_trace.token_positions").is_ok();
+ let has_token_ids = traces_view.leaf_array("activation_trace.token_ids").is_ok();
+ let has_token_positions = traces_view.leaf_array("activation_trace.token_positions").is_ok();
if !has_token_ids || !has_token_positions {
return Ok(Self::handle_missing_tokens(&traces_view));
@@ -428,7 +428,7 @@ impl<B: Backend> TraceValidator<B> {
}
fn handle_missing_tokens(traces_view: &ParameterTree<B::Context>) -> TracerValidationResults {
- if let Ok(expected_logits) = traces_view.leaf("logits") {
+ if let Ok(expected_logits) = traces_view.leaf_array("logits") {
let reference_shape = expected_logits.shape().to_vec();
let metrics = TracerValidationMetrics {
atol: 0.0,
@@ -479,7 +479,7 @@ impl<B: Backend> TraceValidator<B> {
let mut results = Vec::new();
let validate = |path: &str, array: &Ref<Array<B>>| -> Option<TracerValidationResult> {
- if traces_view.leaf(path).is_ok() {
+ if traces_view.leaf_array(path).is_ok() {
Some(TracerValidationResult {
name: path.to_string(),
metrics: Self::validate_array_with_name(data_type, traces_view, path, array),
@@ -549,7 +549,7 @@ impl<B: Backend> TraceValidator<B> {
// Embedding norm (classifier-specific)
if let Some(embedding_norm) = &traces.borrow().embedding_norm {
- if traces_view.leaf("activation_trace.embedding_norm").is_ok() {
+ if traces_view.leaf_array("activation_trace.embedding_norm").is_ok() {
results.push(TracerValidationResult {
name: "activation_trace.embedding_norm".to_string(),
metrics: Self::validate_array_with_name(
@@ -564,7 +564,7 @@ impl<B: Backend> TraceValidator<B> {
// Output pooling (classifier-specific)
if let Some(output_pooling) = &traces.borrow().output_pooling {
- if traces_view.leaf("activation_trace.output_pooling").is_ok() {
+ if traces_view.leaf_array("activation_trace.output_pooling").is_ok() {
results.push(TracerValidationResult {
name: "activation_trace.output_pooling".to_string(),
metrics: Self::validate_array_with_name(
@@ -604,7 +604,7 @@ impl<B: Backend> TraceValidator<B> {
expected_array_path: &str,
produced_array: &Ref<Array<B>>,
) -> TracerValidationMetrics {
- let expected_array = traces_view.leaf(expected_array_path).unwrap();
+ let expected_array = traces_view.leaf_array(expected_array_path).unwrap();
Self::validate_array(data_type, &expected_array, produced_array, None)
}
@@ -783,7 +783,7 @@ impl<B: Backend> TraceValidator<B> {
traces_view: &ParameterTree<B::Context>,
name: &str,
) -> Vec<TargetPrecision> {
- let array = traces_view.leaf(name).unwrap();
+ let array = traces_view.leaf_array(name).unwrap();
let slice = array.as_slice::<SourcePrecision>();
slice.iter().map(|x| NumCast::from(*x).unwrap()).collect()
}
diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rs
index c32a7990c..f9ef609cb 100644
--- a/crates/uzu/tests/parameters_loader_test.rs
+++ b/crates/uzu/tests/parameters_loader_test.rs
@@ -24,6 +24,6 @@ fn test_parameter_loader_basic() {
// tree API check
let subtree = loader.tree().subtree("embedding").unwrap();
- let same = subtree.leaf("weights").unwrap();
+ let same = subtree.leaf_array("weights").unwrap();
assert_eq!(view, same.as_view::<bf16>());
}