Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generalize all remaining encodable blocks (#209)

uuuvn330ebf4d3d67
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index b300da23d..951b554d2 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -3,14 +3,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
use metal::MTLCommandBuffer;
use objc2::{rc::Retained, runtime::ProtocolObject};
-use super::{
- MetalContext,
- encodable_block::{
- ClassifierLayer,
- transformer_layer::{embed_block, linear_block},
- },
- kernel::dsl::SigmoidMetalKernel,
-};
+use super::{MetalContext, kernel::dsl::SigmoidMetalKernel};
use crate::{
DataType,
backends::{
@@ -18,7 +11,10 @@ use crate::{
metal::{Metal, error::ClassifierError},
},
config::{ClassifierModelConfig, ModelMetadata},
- encodable_block::{Activation, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope},
+ encodable_block::{
+ Activation, ClassifierLayer, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope,
+ embed_block, linear_block,
+ },
forward_pass::{
model_shape::ModelShape,
scratch_buffers::ScratchBuffers,
@@ -103,7 +99,7 @@ impl ClassifierContext {
.activation_precision()
.into();
- let embed = embed_block(&decoder_config, &context, &root_loader_view);
+ let embed = embed_block(&decoder_config, context.as_ref(), &root_loader_view);
let global_rope = Self::create_rope_block(&context, data_type, RopeType::Global).map_err(Error::Classifier)?;
let local_rope = classifier_model_config
@@ -205,12 +201,12 @@ impl ClassifierContext {
let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {
Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))
})?;
- let prediction_head_dense = linear_block::<1>(
+ let prediction_head_dense = linear_block::<1, Metal>(
&prediction_head_config.dense_config,
prediction_head_config.use_dense_bias,
model_dim,
[model_dim],
- &context,
+ context.as_ref(),
&prediction_head_dense_tree,
ArrayId::ClassifierPooling,
ArrayId::ClassifierPredictionHeadDense,
@@ -247,12 +243,12 @@ impl ClassifierContext {
let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {
Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))
})?;
- let prediction_head_final_linear = linear_block::<1>(
+ let prediction_head_final_linear = linear_block::<1, Metal>(
&prediction_head_config.readout_config,
true,
model_dim,
[num_labels],
- &context,
+ context.as_ref(),
&prediction_head_readout_tree,
ArrayId::ClassifierPredictionHeadNorm,
ArrayId::ClassifierPredictionHeadLogits,
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
deleted file mode 100644
index 296eda3f1..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
+++ /dev/null
@@ -1,16 +0,0 @@
-//! Embedding encodables (quantized and full-precision).
-
-use super::super::Metal;
-use crate::encodable_block::{
- FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup,
- FullPrecisionEmbeddingReadout as GenericFullPrecisionEmbeddingReadout,
- QuantizedEmbeddingLookup as GenericQuantizedEmbeddingLookup,
- QuantizedEmbeddingReadout as GenericQuantizedEmbeddingReadout,
- QuantizedEmbeddingReadoutError as GenericQuantizedEmbeddingReadoutError,
-};
-
-pub type EmbeddingError = GenericQuantizedEmbeddingReadoutError<Metal>;
-pub type QuantizedEmbeddingReadout = GenericQuantizedEmbeddingReadout<Metal>;
-pub type FullPrecisionEmbeddingReadout = GenericFullPrecisionEmbeddingReadout<Metal>;
-pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;
-pub type QuantizedEmbeddingLookup = GenericQuantizedEmbeddingLookup<Metal>;
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs
deleted file mode 100644
index f393ef8ba..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-use super::super::Metal;
-use crate::encodable_block::{
- FullPrecisionLinear as GenericFullPrecisionLinear, QuantizedLinear as GenericQuantizedLinear,
-};
-
-pub type FullPrecisionLinear = GenericFullPrecisionLinear<Metal>;
-pub type QuantizedLinear = GenericQuantizedLinear<Metal>;
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
deleted file mode 100644
index 155c30798..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-//! Encodable building blocks for classifier and LLM forward passes.
-//!
-//! These are higher-level abstractions that compose kernels into
-//! reusable operations for both classifier and LLM pipelines.
-//!
-//! Encodables implement `EncodableBlock` and orchestrate one or more
-//! kernels to perform operations on `ForwardPassState`.
-
-mod classifier_layer;
-mod decoder;
-mod embedding;
-mod layer;
-mod linear;
-pub mod transformer_layer;
-
-pub use classifier_layer::ClassifierLayer;
-pub use decoder::Decoder;
-pub use embedding::{
- EmbeddingError, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,
- QuantizedEmbeddingReadout,
-};
-pub use layer::LayerExecutables;
-pub use linear::{FullPrecisionLinear, QuantizedLinear};
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
index 147f52c50..20d90de05 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -9,7 +9,7 @@ use std::{
use metal::{MTLBuffer, MTLCommandBuffer, MTLDeviceExt, MTLEvent};
use objc2::{rc::Retained, runtime::ProtocolObject};
-use super::{Decoder, Metal, MetalContext};
+use super::{Metal, MetalContext};
use crate::{
DataType,
backends::{
@@ -22,7 +22,7 @@ use crate::{
metal::kernel::dsl::{MaskUpdateMetalKernel, TokenCopySampledMetalKernel, TokenCopyToResultsMetalKernel},
},
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
- encodable_block::Sampling,
+ encodable_block::{Decoder, Sampling},
forward_pass::{
cache_layers::CacheLayers, model_shape::ModelShape, scratch_buffers::ScratchBuffers, state::SharedBuffers,
},
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 20eb63f8c..dd99c1aa0 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -4,7 +4,6 @@ mod command_buffer;
mod context;
mod copy_encoder;
mod data_type;
-pub mod encodable_block;
pub mod error;
pub mod kernel;
mod language_model_generator_context;
@@ -14,7 +13,6 @@ mod native_buffer;
pub use backend::Metal;
pub use classifier_context::ClassifierContext;
pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MetalContext};
-pub use encodable_block::Decoder;
pub use error::MetalError;
pub use kernel::dsl::MetalKernels;
pub use language_model_generator_context::LanguageModelGeneratorContext;
diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rs
index ac610f7bb..44e22e403 100644
--- a/crates/uzu/src/classifier/mod.rs
+++ b/crates/uzu/src/classifier/mod.rs
@@ -7,7 +7,6 @@ pub use result::{ClassificationOutput, ClassificationStats};
// Re-export context from metal backend
pub use crate::backends::metal::ClassifierContext;
// Re-export encodables from metal backend for convenience
-pub use crate::backends::metal::encodable_block::ClassifierLayer;
-pub use crate::encodable_block::{ClassifierPredictionHead, Pooling};
+pub use crate::encodable_block::{ClassifierLayer, ClassifierPredictionHead, Pooling};
#[cfg(feature = "tracing")]
pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
similarity index 92%
rename from crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
rename to crates/uzu/src/encodable_block/classifier_layer.rs
index 290f445d8..9a5d7cd9c 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -2,18 +2,15 @@ use std::rc::Rc;
use objc2::rc::autoreleasepool;
-use super::transformer_layer;
#[cfg(not(feature = "tracing"))]
use crate::backends::common::CommandBuffer;
use crate::{
DataType,
- backends::{
- common::Backend,
- metal::{Metal, MetalContext},
- },
+ backends::common::{Backend, kernel::matmul::MatmulKernels},
config::TransformerLayerConfig,
encodable_block::{
Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,
+ transformer_layer::{linear_block, mlp_block},
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -38,9 +35,12 @@ pub struct ClassifierLayer<B: Backend> {
mlp_residual_add: Box<dyn EncodableBlock<B>>,
}
-impl ClassifierLayer<Metal> {
+impl<B: Backend + 'static> ClassifierLayer<B>
+where
+ B::Kernels: MatmulKernels,
+{
pub fn new(
- mtl_context: Rc<MetalContext>,
+ context: Rc<B::Context>,
layer_config: &TransformerLayerConfig,
layer_index: usize,
model_dim: usize,
@@ -49,17 +49,17 @@ impl ClassifierLayer<Metal> {
head_dim: usize,
num_groups: usize,
attention_scale: Option<f32>,
- layer_loader: &ParameterTree<MetalContext>,
- rope: Rc<Box<dyn EncodableBlock<Metal>>>,
+ layer_loader: &ParameterTree<B::Context>,
+ rope: Rc<Box<dyn EncodableBlock<B>>>,
) -> Self {
autoreleasepool(|_| {
- let ctx = mtl_context.as_ref(); // Reference for functions expecting &MetalContext
+ let ctx = context.as_ref(); // Reference for functions expecting &B::Context
let attention_config =
layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
- let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorCopy::<Metal>::new(
+ let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>> = Box::new(
+ TensorCopy::<B>::new(
ctx,
intermediate_data_type,
vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
@@ -67,7 +67,7 @@ impl ClassifierLayer<Metal> {
.unwrap(),
);
- let pre_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =
+ let pre_attention_norm: Option<Box<dyn EncodableBlock<B>>> =
if let Some(norm_config) = &layer_config.pre_attention_norm_config {
if layer_loader.subtree("pre_mixer_norm").is_ok() {
Some(Box::new(
@@ -88,7 +88,7 @@ impl ClassifierLayer<Metal> {
None
};
- let qkv_projection = transformer_layer::linear_block(
+ let qkv_projection = linear_block(
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
@@ -100,7 +100,7 @@ impl ClassifierLayer<Metal> {
)
.expect("Failed to create qkv projection");
- let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =
+ let qk_norm: Option<Box<dyn EncodableBlock<B>>> =
if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
match QKNorm::new(
ctx,
@@ -120,7 +120,7 @@ impl ClassifierLayer<Metal> {
None
};
- let out_projection = transformer_layer::linear_block(
+ let out_projection = linear_block(
&attention_config.out_projection_config,
attention_config.has_out_biases,
num_heads * head_dim,
@@ -132,7 +132,7 @@ impl ClassifierLayer<Metal> {
)
.expect("Failed to create out projection");
- let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =
+ let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =
if let Some(norm_config) = &layer_config.post_attention_norm_config {
Some(Box::new(
Normalization::new(
@@ -149,8 +149,8 @@ impl ClassifierLayer<Metal> {
None
};
- let mixer_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorAddSwap::<Metal>::new(
+ let mixer_residual_add: Box<dyn EncodableBlock<B>> = Box::new(
+ TensorAddSwap::<B>::new(
ctx,
intermediate_data_type,
vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
@@ -158,8 +158,8 @@ impl ClassifierLayer<Metal> {
.unwrap(),
);
- let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorCopy::<Metal>::new(
+ let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>> = Box::new(
+ TensorCopy::<B>::new(
ctx,
intermediate_data_type,
vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
@@ -167,7 +167,7 @@ impl ClassifierLayer<Metal> {
.unwrap(),
);
- let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(
+ let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(
Normalization::new(
ctx,
intermediate_data_type,
@@ -179,16 +179,16 @@ impl ClassifierLayer<Metal> {
.expect("Failed to create pre-MLP norm kernel"),
);
- let mlp = transformer_layer::mlp_block(
+ let mlp = mlp_block(
&layer_config.mlp_config,
model_dim,
hidden_dim,
- &mtl_context,
+ context.as_ref(),
&layer_loader.subtree("mlp").unwrap(),
)
.expect("Failed to create mlp block");
- let post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>> =
+ let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =
if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(Box::new(
Normalization::new(
@@ -205,7 +205,7 @@ impl ClassifierLayer<Metal> {
None
};
- let attention: Box<dyn EncodableBlock<Metal>> = Box::new(
+ let attention: Box<dyn EncodableBlock<B>> = Box::new(
Attention::new(
ctx,
intermediate_data_type,
@@ -218,8 +218,8 @@ impl ClassifierLayer<Metal> {
.expect("Failed to create attention kernel"),
);
- let mlp_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorAddSwap::<Metal>::new(
+ let mlp_residual_add: Box<dyn EncodableBlock<B>> = Box::new(
+ TensorAddSwap::<B>::new(
ctx,
intermediate_data_type,
vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
similarity index 84%
rename from crates/uzu/src/backends/metal/encodable_block/decoder.rs
rename to crates/uzu/src/encodable_block/decoder.rs
index ff065fb21..3f0f365b2 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -2,18 +2,14 @@
use std::rc::Rc;
-use super::LayerExecutables;
use crate::{
DataType, DecoderConfig,
- backends::{
- common::Backend,
- metal::{
- Metal, MetalContext,
- encodable_block::transformer_layer::{embed_block, readout_block},
- },
- },
+ backends::common::{Backend, kernel::matmul::MatmulKernels},
config::{DecoderLayerType, MixerConfig},
- encodable_block::{EncodableBlock, EncodingParameters, RMSNorm, Rope},
+ encodable_block::{
+ EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,
+ transformer_layer::{embed_block, readout_block},
+ },
forward_pass::{
model_shape::ModelShape,
state::{ArrayId, ForwardPassState, RopeType},
@@ -31,17 +27,20 @@ pub struct Decoder<B: Backend> {
pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
}
-impl Decoder<Metal> {
+impl<B: Backend + 'static> Decoder<B>
+where
+ B::Kernels: MatmulKernels,
+{
pub fn new(
- mtl_context: Rc<MetalContext>,
+ context: Rc<B::Context>,
decoder_config: Rc<DecoderConfig>,
- root_weight_loader: &ParameterTree<MetalContext>,
+ root_weight_loader: &ParameterTree<B::Context>,
) -> Self {
let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");
- let embed = embed_block(&decoder_config, &mtl_context, root_weight_loader);
+ let embed = embed_block(&decoder_config, context.as_ref(), root_weight_loader);
- let readout = readout_block(&decoder_config, &mtl_context, root_weight_loader);
+ let readout = readout_block(&decoder_config, context.as_ref(), root_weight_loader);
let attention_data_type = Self::attention_data_type(&decoder_config);
let norm_reference_layer =
@@ -59,15 +58,13 @@ impl Decoder<Metal> {
let global_rope = if decoder_config.global_rope_config.is_some() {
attention_data_type
.as_ref()
- .map(|data_type| Self::create_rope_block(&mtl_context, *data_type, RopeType::Global))
+ .map(|data_type| Self::create_rope_block(&context, *data_type, RopeType::Global))
} else {
None
};
let local_rope = if decoder_config.local_rope_config.is_some() {
- attention_data_type
- .as_ref()
- .map(|data_type| Self::create_rope_block(&mtl_context, *data_type, RopeType::Local))
+ attention_data_type.as_ref().map(|data_type| Self::create_rope_block(&context, *data_type, RopeType::Local))
} else {
None
};
@@ -104,7 +101,7 @@ impl Decoder<Metal> {
let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();
LayerExecutables::new(
- mtl_context.clone(),
+ context.clone(),
layer_config,
layer_type,
layer_index,
@@ -120,9 +117,9 @@ impl Decoder<Metal> {
})
.collect::<Vec<_>>();
- let norm_block: Box<dyn EncodableBlock<Metal>> = Box::new(
+ let norm_block: Box<dyn EncodableBlock<B>> = Box::new(
RMSNorm::new(
- mtl_context.as_ref(),
+ context.as_ref(),
norm_data_type,
decoder_config.output_norm_config.clone(),
ArrayId::Main,
@@ -144,12 +141,12 @@ impl Decoder<Metal> {
}
fn create_rope_block(
- mtl_context: &MetalContext,
+ context: &B::Context,
data_type: DataType,
rope_type: RopeType,
- ) -> Rc<Box<dyn EncodableBlock<Metal>>> {
- let rotation: Box<dyn EncodableBlock<Metal>> =
- Box::new(Rope::<Metal>::new(mtl_context, data_type, rope_type).expect("Failed to create Rope"));
+ ) -> Rc<Box<dyn EncodableBlock<B>>> {
+ let rotation: Box<dyn EncodableBlock<B>> =
+ Box::new(Rope::<B>::new(context, data_type, rope_type).expect("Failed to create Rope"));
Rc::new(rotation)
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
similarity index 92%
rename from crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
rename to crates/uzu/src/encodable_block/layer/executables.rs
index ff7922f69..bed8112f2 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -4,19 +4,17 @@ use std::rc::Rc;
use objc2::rc::autoreleasepool;
-use super::{super::transformer_layer, MixerExecutables};
+use super::MixerExecutables;
#[cfg(not(feature = "tracing"))]
use crate::backends::common::CommandBuffer;
use crate::{
DataType, DecoderLayerConfig,
- backends::{
- common::Backend,
- metal::{Metal, MetalContext},
- },
+ backends::common::{Backend, kernel::matmul::MatmulKernels},
config::{DecoderLayerType, MixerConfig},
encodable_block::{
Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,
TensorCopy,
+ transformer_layer::{linear_block, mlp_block},
},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::ParameterTree,
@@ -35,10 +33,13 @@ pub struct LayerExecutables<B: Backend> {
pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,
}
-impl LayerExecutables<Metal> {
+impl<B: Backend + 'static> LayerExecutables<B>
+where
+ B::Kernels: MatmulKernels,
+{
#[allow(clippy::too_many_arguments)]
pub fn new(
- mtl_context: Rc<MetalContext>,
+ context: Rc<B::Context>,
layer_config: &DecoderLayerConfig,
layer_type: &DecoderLayerType,
layer_index: usize,
@@ -48,18 +49,18 @@ impl LayerExecutables<Metal> {
head_dim: usize,
num_groups: usize,
attention_scale: Option<f32>,
- decoder_layer_loader: &ParameterTree<MetalContext>,
- rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,
+ decoder_layer_loader: &ParameterTree<B::Context>,
+ rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
) -> Self {
autoreleasepool(|_| {
- let ctx = mtl_context.as_ref(); // Reference for functions expecting &MetalContext
+ let ctx = context.as_ref(); // Reference for functions expecting &B::Context
let intermediate_data_type: DataType = match &layer_config.mixer_config {
MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),
MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),
MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
};
- let copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorCopy::<Metal>::new(
+ let copy_main_to_shortcut: Box<dyn EncodableBlock<B>> = Box::new(
+ TensorCopy::<B>::new(
ctx,
intermediate_data_type,
vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
@@ -67,7 +68,7 @@ impl LayerExecutables<Metal> {
.unwrap(),
);
- let pre_attention_norm: Box<dyn EncodableBlock<Metal>> = Box::new(
+ let pre_attention_norm: Box<dyn EncodableBlock<B>> = Box::new(
RMSNorm::new(
ctx,
intermediate_data_type,
@@ -83,7 +84,7 @@ impl LayerExecutables<Metal> {
MixerConfig::Attention(attention_config) => {
let rope_block = rope.clone().expect("RoPE encoder missing for attention layer");
- let qkv_projection = transformer_layer::linear_block(
+ let qkv_projection = linear_block(
&attention_config.qkv_projection_config,
attention_config.has_qkv_biases,
model_dim,
@@ -95,7 +96,7 @@ impl LayerExecutables<Metal> {
)
.expect("Failed to create qkv projection");
- let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =
+ let qk_norm: Option<Box<dyn EncodableBlock<B>>> =
if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {
match QKNorm::new(
ctx,
@@ -108,14 +109,14 @@ impl LayerExecutables<Metal> {
num_groups,
head_dim,
) {
- Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<Metal>>),
+ Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<B>>),
Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),
}
} else {
None
};
- let out_projection = transformer_layer::linear_block(
+ let out_projection = linear_block(
&attention_config.out_projection_config,
attention_config.has_out_biases,
num_heads * head_dim,
@@ -137,7 +138,7 @@ impl LayerExecutables<Metal> {
attention_config.is_causal.unwrap_or(true),
attention_config.sliding_window_size,
)
- .expect("Failed to create AttentionWrapper with Metal kernel"),
+ .expect("Failed to create AttentionWrapper kernel"),
);
MixerExecutables::Attention {
@@ -149,7 +150,7 @@ impl LayerExecutables<Metal> {
}
},
MixerConfig::Mamba(mamba_config) => {
- let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(MambaMixer::new(
+ let mixer: Box<dyn EncodableBlock<B>> = Box::new(MambaMixer::new(
ctx,
layer_type.clone(),
mamba_config.clone(),
@@ -165,7 +166,7 @@ impl LayerExecutables<Metal> {
}
},
MixerConfig::ShortConv(short_conv_config) => {
- let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(ShortConvMixer::new(
+ let mixer: Box<dyn EncodableBlock<B>> = Box::new(ShortConvMixer::new(
ctx,
layer_type.clone(),
short_conv_config.clone(),
@@ -179,7 +180,7 @@ impl LayerExecutables<Metal> {
},
};
- let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =
+ let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =
if let Some(norm_config) = &layer_config.post_attention_norm_config {
Some(Box::new(
RMSNorm::new(
@@ -196,8 +197,8 @@ impl LayerExecutables<Metal> {
None
};
- let main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>> = Box::new(
- TensorAddSwap::<Metal>::new(
+ let main_shortcut_add_swap: Box<dyn EncodableBlock<B>> = Box::new(
+ TensorAddSwap::<B>::new(
ctx,
intermediate_data_type,
vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
@@ -205,7 +206,7 @@ impl LayerExecutables<Metal> {
.unwrap(),
);
- let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(
+ let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(
RMSNorm::new(
ctx,
intermediate_data_type,
@@ -217,16 +218,16 @@ impl LayerExecutables<Metal> {
.expect("Failed to create RMS norm kernel"),
);
- let mlp = transformer_layer::mlp_block(
+ let mlp = mlp_block(
&layer_config.mlp_config,
model_dim,
hidden_dim,
- &mtl_context,
+ context.as_ref(),
&decoder_layer_loader.subtree("mlp").unwrap(),
)
.expect("Failed to create mlp block");
- let post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>> =
+ let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =
if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(Box::new(
RMSNorm::new(
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rs
similarity index 100%
rename from crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs
rename to crates/uzu/src/encodable_block/layer/mixer.rs
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs b/crates/uzu/src/encodable_block/layer/mod.rs
similarity index 100%
rename from crates/uzu/src/backends/metal/encodable_block/layer/mod.rs
rename to crates/uzu/src/encodable_block/layer/mod.rs
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index bcaa39755..3807e1dc5 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -5,18 +5,16 @@ use std::env;
use crate::{
Activation, DataType,
array::Array,
- backends::{
- common::{
- Backend, CommandBuffer, Kernels,
- kernel::{
- Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,
- ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
- },
+ backends::common::{
+ Backend, CommandBuffer, Kernels,
+ kernel::{
+ Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,
+ matmul::MatmulKernels,
+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
- metal::{Metal, encodable_block::transformer_layer},
},
config::{DecoderLayerType, Mamba2Config},
- encodable_block::{EncodableBlock, EncodingParameters},
+ encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterTree, resolve_subtree},
};
@@ -39,10 +37,13 @@ pub(crate) struct MambaMixer<B: Backend> {
prefill_mode: SSDPrefillMode,
}
-impl MambaMixer<Metal> {
+impl<B: Backend + 'static> MambaMixer<B>
+where
+ B::Kernels: MatmulKernels,
+{
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
- context: &<Metal as Backend>::Context,
+ context: &B::Context,
layer_type: DecoderLayerType,
mamba_config: Mamba2Config,
layer_index: usize,
@@ -50,7 +51,7 @@ impl MambaMixer<Metal> {
num_heads: usize,
head_dim: usize,
num_groups: usize,
- decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,
+ decoder_layer_loader: &ParameterTree<B::Context>,
) -> Self {
let _ = (num_heads, head_dim, num_groups);
if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {
@@ -61,7 +62,7 @@ impl MambaMixer<Metal> {
let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();
- let in_projection = transformer_layer::linear_block(
+ let in_projection = linear_block(
&mamba_config.in_projection_config,
mamba_config.has_in_biases,
model_dim,
@@ -73,7 +74,7 @@ impl MambaMixer<Metal> {
)
.expect("Failed to create in-projection kernel");
- let out_projection = transformer_layer::linear_block(
+ let out_projection = linear_block(
&mamba_config.out_projection_config,
mamba_config.has_out_biases,
mamba_config.inner_dim(),
@@ -94,26 +95,26 @@ impl MambaMixer<Metal> {
let gate_bias = split_tree.leaf("gate_bias").unwrap().clone();
let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap().clone();
- let split_inproj = <<Metal as Backend>::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)
+ let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)
.expect("Failed to create split in-projection kernel");
- let conv_scan = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(
+ let conv_scan = <B::Kernels as Kernels>::Conv1dScanKernel::new(
context,
data_type,
Self::activation_to_int(&mamba_config.activation),
mamba_config.conv_config.has_biases,
)
.expect("Failed to create conv scan kernel");
- let conv_decode = <<Metal as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(
+ let conv_decode = <B::Kernels as Kernels>::Conv1dDecodeKernel::new(
context,
data_type,
Self::activation_to_int(&mamba_config.activation),
mamba_config.conv_config.has_biases,
)
.expect("Failed to create conv decode kernel");
- let conv_pack = <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)
+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)
.expect("Failed to create conv pack kernel");
let ssd_prefill = SSDPrefillKernels::new(context, data_type).expect("Failed to create SSD prefill kernel");
- let ssd_update = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type)
+ let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type)
.expect("Failed to create SSD decode kernel");
let prefill_mode = resolve_prefill_mode_from_env();
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 9911ca737..15bcfa7d9 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -5,9 +5,12 @@ use crate::{
mod activation;
mod attention;
+mod classifier_layer;
+mod decoder;
mod full_precision_embedding_lookup;
mod full_precision_embedding_readout;
mod full_precision_linear;
+mod layer;
mod layer_norm;
mod mamba_mixer;
mod mlp;
@@ -25,12 +28,16 @@ mod sampling;
mod short_conv_mixer;
mod tensor_add_swap;
mod tensor_copy;
+mod transformer_layer;
pub use activation::Activation;
pub use attention::Attention;
+pub use classifier_layer::ClassifierLayer;
+pub use decoder::Decoder;
pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};
pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};
pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};
+pub use layer::LayerExecutables;
pub use layer_norm::{LayerNorm, LayerNormError};
pub(crate) use mamba_mixer::MambaMixer;
pub use mlp::MlpBlock;
@@ -48,6 +55,7 @@ pub use sampling::Sampling;
pub use short_conv_mixer::ShortConvMixer;
pub use tensor_add_swap::TensorAddSwap;
pub use tensor_copy::TensorCopy;
+pub use transformer_layer::{embed_block, linear_block};
#[derive(Clone)]
pub struct EncodingParameters<'a, B: Backend> {
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index 4582af4e7..f5f1bf1ab 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -1,15 +1,15 @@
use crate::{
DataType,
array::Array,
- backends::{
- common::{
- Backend, CommandBuffer, Context, Kernels,
- kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
+ backends::common::{
+ Backend, CommandBuffer, Context, Kernels,
+ kernel::{
+ ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel,
+ matmul::MatmulKernels,
},
- metal::{Metal, encodable_block::transformer_layer},
},
config::{DecoderLayerType, ShortConvConfig},
- encodable_block::{EncodableBlock, EncodingParameters},
+ encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},
forward_pass::state::{ArrayId, ForwardPassState},
parameters::{ParameterTree, resolve_subtree},
};
@@ -28,14 +28,17 @@ pub struct ShortConvMixer<B: Backend> {
conv_bias: Option<Array<B>>,
}
-impl ShortConvMixer<Metal> {
+impl<B: Backend + 'static> ShortConvMixer<B>
+where
+ B::Kernels: MatmulKernels,
+{
pub fn new(
- context: &<Metal as Backend>::Context,
+ context: &B::Context,
layer_type: DecoderLayerType,
short_conv_config: ShortConvConfig,
layer_index: usize,
model_dim: usize,
- decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,
+ decoder_layer_loader: &ParameterTree<B::Context>,
) -> Self {
if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {
panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);
@@ -46,7 +49,7 @@ impl ShortConvMixer<Metal> {
let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();
- let in_projection = transformer_layer::linear_block(
+ let in_projection = linear_block(
&short_conv_config.in_projection_config,
false,
model_dim,
@@ -58,7 +61,7 @@ impl ShortConvMixer<Metal> {
)
.expect("Failed to create in-projection kernel");
- let out_projection = transformer_layer::linear_block(
+ let out_projection = linear_block(
&short_conv_config.out_projection_config,
false,
model_dim,
@@ -78,17 +81,14 @@ impl ShortConvMixer<Metal> {
};
let has_bias = short_conv_config.conv_config.has_biases;
- let short_conv_pack = <<Metal as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)
+ let short_conv_pack = <B::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)
.expect("Failed to create short conv pack kernel");
- let short_conv_prefill =
- <<Metal as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)
- .expect("Failed to create short conv prefill kernel");
- let short_conv_decode =
- <<Metal as Backend>::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias)
- .expect("Failed to create short conv decode kernel");
- let short_conv_trie =
- <<Metal as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)
- .expect("Failed to create short conv trie kernel");
+ let short_conv_prefill = <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)
+ .expect("Failed to create short conv prefill kernel");
+ let short_conv_decode = <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias)
+ .expect("Failed to create short conv decode kernel");
+ let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)
+ .expect("Failed to create short conv trie kernel");
Self {
layer_index,
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/encodable_block/transformer_layer.rs
similarity index 84%
rename from crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
rename to crates/uzu/src/encodable_block/transformer_layer.rs
index 35ebe71f5..705abd0e1 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/encodable_block/transformer_layer.rs
@@ -1,29 +1,52 @@
+use thiserror::Error;
+
use super::{
FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, QuantizedEmbeddingLookup,
QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
- backends::{
- common::kernel::mlp_gate_act_mul::MlpGateActMulEncodable,
- metal::{Metal, MetalContext, MetalError},
+ backends::common::{
+ Backend,
+ kernel::{matmul::MatmulKernels, mlp_gate_act_mul::MlpGateActMulEncodable},
},
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
- encodable_block::{EncodableBlock, MlpBlock, MoeBlock},
+ encodable_block::{
+ EncodableBlock, FullPrecisionLinearError, MlpBlock, MoeBlock, QuantizedLinearError, moe_block::MoeBlockError,
+ },
forward_pass::state::ArrayId,
- parameters::ParameterTree,
+ parameters::{ParameterLoaderError, ParameterTree},
};
-pub fn linear_block<const N: usize>(
+#[derive(Debug, Error)]
+pub enum LayerError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("QuantizedLinear error: {0}")]
+ QuantizedLinearError(#[source] QuantizedLinearError<B>),
+ #[error("FullPrecisionLinear error: {0}")]
+ FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),
+ #[error("MoeBlock error: {0}")]
+ MoeBlockError(#[source] MoeBlockError<B>),
+ #[error("Parameter loader error: {0}")]
+ ParameterLoaderError(#[source] ParameterLoaderError),
+ #[error("QLoRA linear layer not supported")]
+ QLoRaNotSupported,
+}
+
+pub fn linear_block<const N: usize, B: Backend + 'static>(
config: &LinearConfig,
_has_biases: bool,
input_dimension: usize,
output_dimensions: [usize; N],
- context: &MetalContext,
- parameter_tree: &ParameterTree<MetalContext>,
+ context: &B::Context,
+ parameter_tree: &ParameterTree<B::Context>,
input_array_id: ArrayId,
output_array_id: ArrayId,
-) -> Result<Box<dyn EncodableBlock<Metal>>, MetalError> {
+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>
+where
+ B::Kernels: MatmulKernels,
+{
let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
@@ -36,7 +59,7 @@ pub fn linear_block<const N: usize>(
input_array_id,
output_array_id,
)
- .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;
+ .map_err(LayerError::QuantizedLinearError)?;
Ok(Box::new(block))
},
LinearConfig::FullPrecision {
@@ -51,23 +74,26 @@ pub fn linear_block<const N: usize>(
input_array_id,
output_array_id,
)
- .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;
+ .map_err(LayerError::FullPrecisionLinearError)?;
Ok(Box::new(block))
},
LinearConfig::QLoRA {
..
- } => Err(MetalError::Generic("QLoRA linear layer not supported".to_string())),
+ } => Err(LayerError::QLoRaNotSupported),
}
}
/// Creates an MLP block using the unfused implementation (separate up, activation, down)
-pub fn mlp_block(
+pub fn mlp_block<B: Backend + 'static>(
config: &MLPConfig,
model_dimension: usize,
hidden_dimension: usize,
- context: &MetalContext,
- parameter_tree: &ParameterTree<MetalContext>,
-) -> Result<Box<dyn EncodableBlock<Metal>>, MetalError> {
+ context: &B::Context,
+ parameter_tree: &ParameterTree<B::Context>,
+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>
+where
+ B::Kernels: MatmulKernels,
+{
if let crate::config::MLPConfig::Dense(dense_config) = config {
let data_type: DataType = dense_config.linear_config.activation_precision().into();
@@ -78,14 +104,15 @@ pub fn mlp_block(
model_dimension,
[2 * hidden_dimension],
context,
- &parameter_tree.subtree("up_projection").map_err(|error| MetalError::Generic(format!("{:?}", error)))?,
+ &parameter_tree.subtree("up_projection").map_err(LayerError::ParameterLoaderError)?,
ArrayId::Main,
ArrayId::MlpFusedUp,
)?;
// Gate activation + multiply
let gate_activation =
- MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)?;
+ MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)
+ .map_err(LayerError::BackendError)?;
// Down projection
let down_projection = linear_block(
@@ -94,7 +121,7 @@ pub fn mlp_block(
hidden_dimension,
[model_dimension],
context,
- &parameter_tree.subtree("down_projection").map_err(|error| MetalError::Generic(format!("{:?}", error)))?,
+ &parameter_tree.subtree("down_projection").map_err(LayerError::ParameterLoaderError)?,
ArrayId::MlpHidden,
ArrayId::Main,
)?;
@@ -105,18 +132,18 @@ pub fn mlp_block(
if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
let mixture_of_experts_block =
MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)
- .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;
+ .map_err(LayerError::MoeBlockError)?;
return Ok(Box::new(mixture_of_experts_block));
}
unreachable!("Unknown MLP config")
}
-pub fn embed_block(
+pub fn embed_block<B: Backend + 'static>(
config: &DecoderConfig,
- context: &MetalContext,
- parameter_tree: &ParameterTree<MetalContext>,
-) -> Box<dyn EncodableBlock<Metal>> {
+ context: &B::Context,
+ parameter_tree: &ParameterTree<B::Context>,
+) -> Box<dyn EncodableBlock<B>> {
match &config.embedding_config {
EmbeddingConfig::Tied {
common,
@@ -252,11 +279,14 @@ pub fn embed_block(
}
}
-pub fn readout_block(
+pub fn readout_block<B: Backend + 'static>(
config: &DecoderConfig,
- context: &MetalContext,
- parameter_tree: &ParameterTree<MetalContext>,
-) -> Box<dyn EncodableBlock<Metal>> {
+ context: &B::Context,
+ parameter_tree: &ParameterTree<B::Context>,
+) -> Box<dyn EncodableBlock<B>>
+where
+ B::Kernels: MatmulKernels,
+{
match &config.embedding_config {
EmbeddingConfig::Tied {
precision,