A high-performance inference engine for AI models
generalize all remaining encodable blocks (#209)
uuuvn330ebf4d3d67
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex b300da23d..951b554d2 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -3,14 +3,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};use metal::MTLCommandBuffer;use objc2::{rc::Retained, runtime::ProtocolObject};-use super::{- MetalContext,- encodable_block::{- ClassifierLayer,- transformer_layer::{embed_block, linear_block},- },- kernel::dsl::SigmoidMetalKernel,-};+use super::{MetalContext, kernel::dsl::SigmoidMetalKernel};use crate::{DataType,backends::{@@ -18,7 +11,10 @@ use crate::{metal::{Metal, error::ClassifierError},},config::{ClassifierModelConfig, ModelMetadata},- encodable_block::{Activation, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope},+ encodable_block::{+ Activation, ClassifierLayer, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope,+ embed_block, linear_block,+ },forward_pass::{model_shape::ModelShape,scratch_buffers::ScratchBuffers,@@ -103,7 +99,7 @@ impl ClassifierContext {.activation_precision().into();- let embed = embed_block(&decoder_config, &context, &root_loader_view);+ let embed = embed_block(&decoder_config, context.as_ref(), &root_loader_view);let global_rope = Self::create_rope_block(&context, data_type, RopeType::Global).map_err(Error::Classifier)?;let local_rope = classifier_model_config@@ -205,12 +201,12 @@ impl ClassifierContext {let prediction_head_dense_tree = prediction_head_tree.subtree("dense").map_err(|_| {Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.dense".to_string()))})?;- let prediction_head_dense = linear_block::<1>(+ let prediction_head_dense = linear_block::<1, Metal>(&prediction_head_config.dense_config,prediction_head_config.use_dense_bias,model_dim,[model_dim],- &context,+ context.as_ref(),&prediction_head_dense_tree,ArrayId::ClassifierPooling,ArrayId::ClassifierPredictionHeadDense,@@ -247,12 +243,12 @@ impl ClassifierContext {let prediction_head_readout_tree = prediction_head_tree.subtree("readout").map_err(|_| {Error::Classifier(ClassifierError::WeightSubtreeNotFound("prediction_head.readout".to_string()))})?;- let prediction_head_final_linear = linear_block::<1>(+ let prediction_head_final_linear = linear_block::<1, Metal>(&prediction_head_config.readout_config,true,model_dim,[num_labels],- &context,+ context.as_ref(),&prediction_head_readout_tree,ArrayId::ClassifierPredictionHeadNorm,ArrayId::ClassifierPredictionHeadLogits,diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rsdeleted file mode 100644index 296eda3f1..000000000--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs+++ /dev/null@@ -1,16 +0,0 @@-//! Embedding encodables (quantized and full-precision).--use super::super::Metal;-use crate::encodable_block::{- FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup,- FullPrecisionEmbeddingReadout as GenericFullPrecisionEmbeddingReadout,- QuantizedEmbeddingLookup as GenericQuantizedEmbeddingLookup,- QuantizedEmbeddingReadout as GenericQuantizedEmbeddingReadout,- QuantizedEmbeddingReadoutError as GenericQuantizedEmbeddingReadoutError,-};--pub type EmbeddingError = GenericQuantizedEmbeddingReadoutError<Metal>;-pub type QuantizedEmbeddingReadout = GenericQuantizedEmbeddingReadout<Metal>;-pub type FullPrecisionEmbeddingReadout = GenericFullPrecisionEmbeddingReadout<Metal>;-pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;-pub type QuantizedEmbeddingLookup = GenericQuantizedEmbeddingLookup<Metal>;diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rsdeleted file mode 100644index f393ef8ba..000000000--- a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs+++ /dev/null@@ -1,7 +0,0 @@-use super::super::Metal;-use crate::encodable_block::{- FullPrecisionLinear as GenericFullPrecisionLinear, QuantizedLinear as GenericQuantizedLinear,-};--pub type FullPrecisionLinear = GenericFullPrecisionLinear<Metal>;-pub type QuantizedLinear = GenericQuantizedLinear<Metal>;diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsdeleted file mode 100644index 155c30798..000000000--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ /dev/null@@ -1,23 +0,0 @@-//! Encodable building blocks for classifier and LLM forward passes.-//!-//! These are higher-level abstractions that compose kernels into-//! reusable operations for both classifier and LLM pipelines.-//!-//! Encodables implement `EncodableBlock` and orchestrate one or more-//! kernels to perform operations on `ForwardPassState`.--mod classifier_layer;-mod decoder;-mod embedding;-mod layer;-mod linear;-pub mod transformer_layer;--pub use classifier_layer::ClassifierLayer;-pub use decoder::Decoder;-pub use embedding::{- EmbeddingError, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,- QuantizedEmbeddingReadout,-};-pub use layer::LayerExecutables;-pub use linear::{FullPrecisionLinear, QuantizedLinear};diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rsindex 147f52c50..20d90de05 100644--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs@@ -9,7 +9,7 @@ use std::{use metal::{MTLBuffer, MTLCommandBuffer, MTLDeviceExt, MTLEvent};use objc2::{rc::Retained, runtime::ProtocolObject};-use super::{Decoder, Metal, MetalContext};+use super::{Metal, MetalContext};use crate::{DataType,backends::{@@ -22,7 +22,7 @@ use crate::{metal::kernel::dsl::{MaskUpdateMetalKernel, TokenCopySampledMetalKernel, TokenCopyToResultsMetalKernel},},config::{DecoderConfig, LanguageModelConfig, ModelMetadata},- encodable_block::Sampling,+ encodable_block::{Decoder, Sampling},forward_pass::{cache_layers::CacheLayers, model_shape::ModelShape, scratch_buffers::ScratchBuffers, state::SharedBuffers,},diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 20eb63f8c..dd99c1aa0 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -4,7 +4,6 @@ mod command_buffer;mod context;mod copy_encoder;mod data_type;-pub mod encodable_block;pub mod error;pub mod kernel;mod language_model_generator_context;@@ -14,7 +13,6 @@ mod native_buffer;pub use backend::Metal;pub use classifier_context::ClassifierContext;pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MetalContext};-pub use encodable_block::Decoder;pub use error::MetalError;pub use kernel::dsl::MetalKernels;pub use language_model_generator_context::LanguageModelGeneratorContext;diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rsindex ac610f7bb..44e22e403 100644--- a/crates/uzu/src/classifier/mod.rs+++ b/crates/uzu/src/classifier/mod.rs@@ -7,7 +7,6 @@ pub use result::{ClassificationOutput, ClassificationStats};// Re-export context from metal backendpub use crate::backends::metal::ClassifierContext;// Re-export encodables from metal backend for convenience-pub use crate::backends::metal::encodable_block::ClassifierLayer;-pub use crate::encodable_block::{ClassifierPredictionHead, Pooling};+pub use crate::encodable_block::{ClassifierLayer, ClassifierPredictionHead, Pooling};#[cfg(feature = "tracing")]pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rssimilarity index 92%rename from crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsrename to crates/uzu/src/encodable_block/classifier_layer.rsindex 290f445d8..9a5d7cd9c 100644--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -2,18 +2,15 @@ use std::rc::Rc;use objc2::rc::autoreleasepool;-use super::transformer_layer;#[cfg(not(feature = "tracing"))]use crate::backends::common::CommandBuffer;use crate::{DataType,- backends::{- common::Backend,- metal::{Metal, MetalContext},- },+ backends::common::{Backend, kernel::matmul::MatmulKernels},config::TransformerLayerConfig,encodable_block::{Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,+ transformer_layer::{linear_block, mlp_block},},forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -38,9 +35,12 @@ pub struct ClassifierLayer<B: Backend> {mlp_residual_add: Box<dyn EncodableBlock<B>>,}-impl ClassifierLayer<Metal> {+impl<B: Backend + 'static> ClassifierLayer<B>+where+ B::Kernels: MatmulKernels,+{pub fn new(- mtl_context: Rc<MetalContext>,+ context: Rc<B::Context>,layer_config: &TransformerLayerConfig,layer_index: usize,model_dim: usize,@@ -49,17 +49,17 @@ impl ClassifierLayer<Metal> {head_dim: usize,num_groups: usize,attention_scale: Option<f32>,- layer_loader: &ParameterTree<MetalContext>,- rope: Rc<Box<dyn EncodableBlock<Metal>>>,+ layer_loader: &ParameterTree<B::Context>,+ rope: Rc<Box<dyn EncodableBlock<B>>>,) -> Self {autoreleasepool(|_| {- let ctx = mtl_context.as_ref(); // Reference for functions expecting &MetalContext+ let ctx = context.as_ref(); // Reference for functions expecting &B::Contextlet attention_config =layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();- let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorCopy::<Metal>::new(+ let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>> = Box::new(+ TensorCopy::<B>::new(ctx,intermediate_data_type,vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),@@ -67,7 +67,7 @@ impl ClassifierLayer<Metal> {.unwrap(),);- let pre_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =+ let pre_attention_norm: Option<Box<dyn EncodableBlock<B>>> =if let Some(norm_config) = &layer_config.pre_attention_norm_config {if layer_loader.subtree("pre_mixer_norm").is_ok() {Some(Box::new(@@ -88,7 +88,7 @@ impl ClassifierLayer<Metal> {None};- let qkv_projection = transformer_layer::linear_block(+ let qkv_projection = linear_block(&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,@@ -100,7 +100,7 @@ impl ClassifierLayer<Metal> {).expect("Failed to create qkv projection");- let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =+ let qk_norm: Option<Box<dyn EncodableBlock<B>>> =if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {match QKNorm::new(ctx,@@ -120,7 +120,7 @@ impl ClassifierLayer<Metal> {None};- let out_projection = transformer_layer::linear_block(+ let out_projection = linear_block(&attention_config.out_projection_config,attention_config.has_out_biases,num_heads * head_dim,@@ -132,7 +132,7 @@ impl ClassifierLayer<Metal> {).expect("Failed to create out projection");- let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =+ let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =if let Some(norm_config) = &layer_config.post_attention_norm_config {Some(Box::new(Normalization::new(@@ -149,8 +149,8 @@ impl ClassifierLayer<Metal> {None};- let mixer_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorAddSwap::<Metal>::new(+ let mixer_residual_add: Box<dyn EncodableBlock<B>> = Box::new(+ TensorAddSwap::<B>::new(ctx,intermediate_data_type,vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),@@ -158,8 +158,8 @@ impl ClassifierLayer<Metal> {.unwrap(),);- let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorCopy::<Metal>::new(+ let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>> = Box::new(+ TensorCopy::<B>::new(ctx,intermediate_data_type,vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),@@ -167,7 +167,7 @@ impl ClassifierLayer<Metal> {.unwrap(),);- let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(+ let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(Normalization::new(ctx,intermediate_data_type,@@ -179,16 +179,16 @@ impl ClassifierLayer<Metal> {.expect("Failed to create pre-MLP norm kernel"),);- let mlp = transformer_layer::mlp_block(+ let mlp = mlp_block(&layer_config.mlp_config,model_dim,hidden_dim,- &mtl_context,+ context.as_ref(),&layer_loader.subtree("mlp").unwrap(),).expect("Failed to create mlp block");- let post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>> =+ let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Box::new(Normalization::new(@@ -205,7 +205,7 @@ impl ClassifierLayer<Metal> {None};- let attention: Box<dyn EncodableBlock<Metal>> = Box::new(+ let attention: Box<dyn EncodableBlock<B>> = Box::new(Attention::new(ctx,intermediate_data_type,@@ -218,8 +218,8 @@ impl ClassifierLayer<Metal> {.expect("Failed to create attention kernel"),);- let mlp_residual_add: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorAddSwap::<Metal>::new(+ let mlp_residual_add: Box<dyn EncodableBlock<B>> = Box::new(+ TensorAddSwap::<B>::new(ctx,intermediate_data_type,vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rssimilarity index 84%rename from crates/uzu/src/backends/metal/encodable_block/decoder.rsrename to crates/uzu/src/encodable_block/decoder.rsindex ff065fb21..3f0f365b2 100644--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -2,18 +2,14 @@use std::rc::Rc;-use super::LayerExecutables;use crate::{DataType, DecoderConfig,- backends::{- common::Backend,- metal::{- Metal, MetalContext,- encodable_block::transformer_layer::{embed_block, readout_block},- },- },+ backends::common::{Backend, kernel::matmul::MatmulKernels},config::{DecoderLayerType, MixerConfig},- encodable_block::{EncodableBlock, EncodingParameters, RMSNorm, Rope},+ encodable_block::{+ EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,+ transformer_layer::{embed_block, readout_block},+ },forward_pass::{model_shape::ModelShape,state::{ArrayId, ForwardPassState, RopeType},@@ -31,17 +27,20 @@ pub struct Decoder<B: Backend> {pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,}-impl Decoder<Metal> {+impl<B: Backend + 'static> Decoder<B>+where+ B::Kernels: MatmulKernels,+{pub fn new(- mtl_context: Rc<MetalContext>,+ context: Rc<B::Context>,decoder_config: Rc<DecoderConfig>,- root_weight_loader: &ParameterTree<MetalContext>,+ root_weight_loader: &ParameterTree<B::Context>,) -> Self {let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");- let embed = embed_block(&decoder_config, &mtl_context, root_weight_loader);+ let embed = embed_block(&decoder_config, context.as_ref(), root_weight_loader);- let readout = readout_block(&decoder_config, &mtl_context, root_weight_loader);+ let readout = readout_block(&decoder_config, context.as_ref(), root_weight_loader);let attention_data_type = Self::attention_data_type(&decoder_config);let norm_reference_layer =@@ -59,15 +58,13 @@ impl Decoder<Metal> {let global_rope = if decoder_config.global_rope_config.is_some() {attention_data_type.as_ref()- .map(|data_type| Self::create_rope_block(&mtl_context, *data_type, RopeType::Global))+ .map(|data_type| Self::create_rope_block(&context, *data_type, RopeType::Global))} else {None};let local_rope = if decoder_config.local_rope_config.is_some() {- attention_data_type- .as_ref()- .map(|data_type| Self::create_rope_block(&mtl_context, *data_type, RopeType::Local))+ attention_data_type.as_ref().map(|data_type| Self::create_rope_block(&context, *data_type, RopeType::Local))} else {None};@@ -104,7 +101,7 @@ impl Decoder<Metal> {let layer_loader = decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap();LayerExecutables::new(- mtl_context.clone(),+ context.clone(),layer_config,layer_type,layer_index,@@ -120,9 +117,9 @@ impl Decoder<Metal> {}).collect::<Vec<_>>();- let norm_block: Box<dyn EncodableBlock<Metal>> = Box::new(+ let norm_block: Box<dyn EncodableBlock<B>> = Box::new(RMSNorm::new(- mtl_context.as_ref(),+ context.as_ref(),norm_data_type,decoder_config.output_norm_config.clone(),ArrayId::Main,@@ -144,12 +141,12 @@ impl Decoder<Metal> {}fn create_rope_block(- mtl_context: &MetalContext,+ context: &B::Context,data_type: DataType,rope_type: RopeType,- ) -> Rc<Box<dyn EncodableBlock<Metal>>> {- let rotation: Box<dyn EncodableBlock<Metal>> =- Box::new(Rope::<Metal>::new(mtl_context, data_type, rope_type).expect("Failed to create Rope"));+ ) -> Rc<Box<dyn EncodableBlock<B>>> {+ let rotation: Box<dyn EncodableBlock<B>> =+ Box::new(Rope::<B>::new(context, data_type, rope_type).expect("Failed to create Rope"));Rc::new(rotation)}diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rssimilarity index 92%rename from crates/uzu/src/backends/metal/encodable_block/layer/executables.rsrename to crates/uzu/src/encodable_block/layer/executables.rsindex ff7922f69..bed8112f2 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -4,19 +4,17 @@ use std::rc::Rc;use objc2::rc::autoreleasepool;-use super::{super::transformer_layer, MixerExecutables};+use super::MixerExecutables;#[cfg(not(feature = "tracing"))]use crate::backends::common::CommandBuffer;use crate::{DataType, DecoderLayerConfig,- backends::{- common::Backend,- metal::{Metal, MetalContext},- },+ backends::common::{Backend, kernel::matmul::MatmulKernels},config::{DecoderLayerType, MixerConfig},encodable_block::{Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,TensorCopy,+ transformer_layer::{linear_block, mlp_block},},forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,@@ -35,10 +33,13 @@ pub struct LayerExecutables<B: Backend> {pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,}-impl LayerExecutables<Metal> {+impl<B: Backend + 'static> LayerExecutables<B>+where+ B::Kernels: MatmulKernels,+{#[allow(clippy::too_many_arguments)]pub fn new(- mtl_context: Rc<MetalContext>,+ context: Rc<B::Context>,layer_config: &DecoderLayerConfig,layer_type: &DecoderLayerType,layer_index: usize,@@ -48,18 +49,18 @@ impl LayerExecutables<Metal> {head_dim: usize,num_groups: usize,attention_scale: Option<f32>,- decoder_layer_loader: &ParameterTree<MetalContext>,- rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,+ decoder_layer_loader: &ParameterTree<B::Context>,+ rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,) -> Self {autoreleasepool(|_| {- let ctx = mtl_context.as_ref(); // Reference for functions expecting &MetalContext+ let ctx = context.as_ref(); // Reference for functions expecting &B::Contextlet intermediate_data_type: DataType = match &layer_config.mixer_config {MixerConfig::Attention(attention) => attention.qkv_projection_config.activation_precision().into(),MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),};- let copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorCopy::<Metal>::new(+ let copy_main_to_shortcut: Box<dyn EncodableBlock<B>> = Box::new(+ TensorCopy::<B>::new(ctx,intermediate_data_type,vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),@@ -67,7 +68,7 @@ impl LayerExecutables<Metal> {.unwrap(),);- let pre_attention_norm: Box<dyn EncodableBlock<Metal>> = Box::new(+ let pre_attention_norm: Box<dyn EncodableBlock<B>> = Box::new(RMSNorm::new(ctx,intermediate_data_type,@@ -83,7 +84,7 @@ impl LayerExecutables<Metal> {MixerConfig::Attention(attention_config) => {let rope_block = rope.clone().expect("RoPE encoder missing for attention layer");- let qkv_projection = transformer_layer::linear_block(+ let qkv_projection = linear_block(&attention_config.qkv_projection_config,attention_config.has_qkv_biases,model_dim,@@ -95,7 +96,7 @@ impl LayerExecutables<Metal> {).expect("Failed to create qkv projection");- let qk_norm: Option<Box<dyn EncodableBlock<Metal>>> =+ let qk_norm: Option<Box<dyn EncodableBlock<B>>> =if attention_config.query_norm_config.is_some() || attention_config.key_norm_config.is_some() {match QKNorm::new(ctx,@@ -108,14 +109,14 @@ impl LayerExecutables<Metal> {num_groups,head_dim,) {- Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<Metal>>),+ Ok(qk_norm) => Some(Box::new(qk_norm) as Box<dyn EncodableBlock<B>>),Err(e) => panic!("Failed to create QK norm kernel for layer {}: {:?}", layer_index, e),}} else {None};- let out_projection = transformer_layer::linear_block(+ let out_projection = linear_block(&attention_config.out_projection_config,attention_config.has_out_biases,num_heads * head_dim,@@ -137,7 +138,7 @@ impl LayerExecutables<Metal> {attention_config.is_causal.unwrap_or(true),attention_config.sliding_window_size,)- .expect("Failed to create AttentionWrapper with Metal kernel"),+ .expect("Failed to create AttentionWrapper kernel"),);MixerExecutables::Attention {@@ -149,7 +150,7 @@ impl LayerExecutables<Metal> {}},MixerConfig::Mamba(mamba_config) => {- let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(MambaMixer::new(+ let mixer: Box<dyn EncodableBlock<B>> = Box::new(MambaMixer::new(ctx,layer_type.clone(),mamba_config.clone(),@@ -165,7 +166,7 @@ impl LayerExecutables<Metal> {}},MixerConfig::ShortConv(short_conv_config) => {- let mixer: Box<dyn EncodableBlock<Metal>> = Box::new(ShortConvMixer::new(+ let mixer: Box<dyn EncodableBlock<B>> = Box::new(ShortConvMixer::new(ctx,layer_type.clone(),short_conv_config.clone(),@@ -179,7 +180,7 @@ impl LayerExecutables<Metal> {},};- let post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>> =+ let post_attention_norm: Option<Box<dyn EncodableBlock<B>>> =if let Some(norm_config) = &layer_config.post_attention_norm_config {Some(Box::new(RMSNorm::new(@@ -196,8 +197,8 @@ impl LayerExecutables<Metal> {None};- let main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>> = Box::new(- TensorAddSwap::<Metal>::new(+ let main_shortcut_add_swap: Box<dyn EncodableBlock<B>> = Box::new(+ TensorAddSwap::<B>::new(ctx,intermediate_data_type,vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),@@ -205,7 +206,7 @@ impl LayerExecutables<Metal> {.unwrap(),);- let pre_mlp_norm: Box<dyn EncodableBlock<Metal>> = Box::new(+ let pre_mlp_norm: Box<dyn EncodableBlock<B>> = Box::new(RMSNorm::new(ctx,intermediate_data_type,@@ -217,16 +218,16 @@ impl LayerExecutables<Metal> {.expect("Failed to create RMS norm kernel"),);- let mlp = transformer_layer::mlp_block(+ let mlp = mlp_block(&layer_config.mlp_config,model_dim,hidden_dim,- &mtl_context,+ context.as_ref(),&decoder_layer_loader.subtree("mlp").unwrap(),).expect("Failed to create mlp block");- let post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>> =+ let post_mlp_norm: Option<Box<dyn EncodableBlock<B>>> =if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Box::new(RMSNorm::new(diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/encodable_block/layer/mixer.rssimilarity index 100%rename from crates/uzu/src/backends/metal/encodable_block/layer/mixer.rsrename to crates/uzu/src/encodable_block/layer/mixer.rsdiff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mod.rs b/crates/uzu/src/encodable_block/layer/mod.rssimilarity index 100%rename from crates/uzu/src/backends/metal/encodable_block/layer/mod.rsrename to crates/uzu/src/encodable_block/layer/mod.rsdiff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex bcaa39755..3807e1dc5 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -5,18 +5,16 @@ use std::env;use crate::{Activation, DataType,array::Array,- backends::{- common::{- Backend, CommandBuffer, Kernels,- kernel::{- Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,- ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},- },+ backends::common::{+ Backend, CommandBuffer, Kernels,+ kernel::{+ Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,+ matmul::MatmulKernels,+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},- metal::{Metal, encodable_block::transformer_layer},},config::{DecoderLayerType, Mamba2Config},- encodable_block::{EncodableBlock, EncodingParameters},+ encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterTree, resolve_subtree},};@@ -39,10 +37,13 @@ pub(crate) struct MambaMixer<B: Backend> {prefill_mode: SSDPrefillMode,}-impl MambaMixer<Metal> {+impl<B: Backend + 'static> MambaMixer<B>+where+ B::Kernels: MatmulKernels,+{#[allow(clippy::too_many_arguments)]pub(crate) fn new(- context: &<Metal as Backend>::Context,+ context: &B::Context,layer_type: DecoderLayerType,mamba_config: Mamba2Config,layer_index: usize,@@ -50,7 +51,7 @@ impl MambaMixer<Metal> {num_heads: usize,head_dim: usize,num_groups: usize,- decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,+ decoder_layer_loader: &ParameterTree<B::Context>,) -> Self {let _ = (num_heads, head_dim, num_groups);if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {@@ -61,7 +62,7 @@ impl MambaMixer<Metal> {let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();- let in_projection = transformer_layer::linear_block(+ let in_projection = linear_block(&mamba_config.in_projection_config,mamba_config.has_in_biases,model_dim,@@ -73,7 +74,7 @@ impl MambaMixer<Metal> {).expect("Failed to create in-projection kernel");- let out_projection = transformer_layer::linear_block(+ let out_projection = linear_block(&mamba_config.out_projection_config,mamba_config.has_out_biases,mamba_config.inner_dim(),@@ -94,26 +95,26 @@ impl MambaMixer<Metal> {let gate_bias = split_tree.leaf("gate_bias").unwrap().clone();let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap().clone();- let split_inproj = <<Metal as Backend>::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)+ let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type).expect("Failed to create split in-projection kernel");- let conv_scan = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(+ let conv_scan = <B::Kernels as Kernels>::Conv1dScanKernel::new(context,data_type,Self::activation_to_int(&mamba_config.activation),mamba_config.conv_config.has_biases,).expect("Failed to create conv scan kernel");- let conv_decode = <<Metal as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(+ let conv_decode = <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context,data_type,Self::activation_to_int(&mamba_config.activation),mamba_config.conv_config.has_biases,).expect("Failed to create conv decode kernel");- let conv_pack = <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)+ let conv_pack = <B::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type).expect("Failed to create conv pack kernel");let ssd_prefill = SSDPrefillKernels::new(context, data_type).expect("Failed to create SSD prefill kernel");- let ssd_update = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type)+ let ssd_update = <B::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type).expect("Failed to create SSD decode kernel");let prefill_mode = resolve_prefill_mode_from_env();diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 9911ca737..15bcfa7d9 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -5,9 +5,12 @@ use crate::{mod activation;mod attention;+mod classifier_layer;+mod decoder;mod full_precision_embedding_lookup;mod full_precision_embedding_readout;mod full_precision_linear;+mod layer;mod layer_norm;mod mamba_mixer;mod mlp;@@ -25,12 +28,16 @@ mod sampling;mod short_conv_mixer;mod tensor_add_swap;mod tensor_copy;+mod transformer_layer;pub use activation::Activation;pub use attention::Attention;+pub use classifier_layer::ClassifierLayer;+pub use decoder::Decoder;pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};+pub use layer::LayerExecutables;pub use layer_norm::{LayerNorm, LayerNormError};pub(crate) use mamba_mixer::MambaMixer;pub use mlp::MlpBlock;@@ -48,6 +55,7 @@ pub use sampling::Sampling;pub use short_conv_mixer::ShortConvMixer;pub use tensor_add_swap::TensorAddSwap;pub use tensor_copy::TensorCopy;+pub use transformer_layer::{embed_block, linear_block};#[derive(Clone)]pub struct EncodingParameters<'a, B: Backend> {diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex 4582af4e7..f5f1bf1ab 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -1,15 +1,15 @@use crate::{DataType,array::Array,- backends::{- common::{- Backend, CommandBuffer, Context, Kernels,- kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},+ backends::common::{+ Backend, CommandBuffer, Context, Kernels,+ kernel::{+ ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel,+ matmul::MatmulKernels,},- metal::{Metal, encodable_block::transformer_layer},},config::{DecoderLayerType, ShortConvConfig},- encodable_block::{EncodableBlock, EncodingParameters},+ encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},forward_pass::state::{ArrayId, ForwardPassState},parameters::{ParameterTree, resolve_subtree},};@@ -28,14 +28,17 @@ pub struct ShortConvMixer<B: Backend> {conv_bias: Option<Array<B>>,}-impl ShortConvMixer<Metal> {+impl<B: Backend + 'static> ShortConvMixer<B>+where+ B::Kernels: MatmulKernels,+{pub fn new(- context: &<Metal as Backend>::Context,+ context: &B::Context,layer_type: DecoderLayerType,short_conv_config: ShortConvConfig,layer_index: usize,model_dim: usize,- decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,+ decoder_layer_loader: &ParameterTree<B::Context>,) -> Self {if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);@@ -46,7 +49,7 @@ impl ShortConvMixer<Metal> {let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();- let in_projection = transformer_layer::linear_block(+ let in_projection = linear_block(&short_conv_config.in_projection_config,false,model_dim,@@ -58,7 +61,7 @@ impl ShortConvMixer<Metal> {).expect("Failed to create in-projection kernel");- let out_projection = transformer_layer::linear_block(+ let out_projection = linear_block(&short_conv_config.out_projection_config,false,model_dim,@@ -78,17 +81,14 @@ impl ShortConvMixer<Metal> {};let has_bias = short_conv_config.conv_config.has_biases;- let short_conv_pack = <<Metal as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)+ let short_conv_pack = <B::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type).expect("Failed to create short conv pack kernel");- let short_conv_prefill =- <<Metal as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)- .expect("Failed to create short conv prefill kernel");- let short_conv_decode =- <<Metal as Backend>::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias)- .expect("Failed to create short conv decode kernel");- let short_conv_trie =- <<Metal as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)- .expect("Failed to create short conv trie kernel");+ let short_conv_prefill = <B::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)+ .expect("Failed to create short conv prefill kernel");+ let short_conv_decode = <B::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias)+ .expect("Failed to create short conv decode kernel");+ let short_conv_trie = <B::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)+ .expect("Failed to create short conv trie kernel");Self {layer_index,diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/encodable_block/transformer_layer.rssimilarity index 84%rename from crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsrename to crates/uzu/src/encodable_block/transformer_layer.rsindex 35ebe71f5..705abd0e1 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/encodable_block/transformer_layer.rs@@ -1,29 +1,52 @@+use thiserror::Error;+use super::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, QuantizedEmbeddingLookup,QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,- backends::{- common::kernel::mlp_gate_act_mul::MlpGateActMulEncodable,- metal::{Metal, MetalContext, MetalError},+ backends::common::{+ Backend,+ kernel::{matmul::MatmulKernels, mlp_gate_act_mul::MlpGateActMulEncodable},},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},- encodable_block::{EncodableBlock, MlpBlock, MoeBlock},+ encodable_block::{+ EncodableBlock, FullPrecisionLinearError, MlpBlock, MoeBlock, QuantizedLinearError, moe_block::MoeBlockError,+ },forward_pass::state::ArrayId,- parameters::ParameterTree,+ parameters::{ParameterLoaderError, ParameterTree},};-pub fn linear_block<const N: usize>(+#[derive(Debug, Error)]+pub enum LayerError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("QuantizedLinear error: {0}")]+ QuantizedLinearError(#[source] QuantizedLinearError<B>),+ #[error("FullPrecisionLinear error: {0}")]+ FullPrecisionLinearError(#[source] FullPrecisionLinearError<B>),+ #[error("MoeBlock error: {0}")]+ MoeBlockError(#[source] MoeBlockError<B>),+ #[error("Parameter loader error: {0}")]+ ParameterLoaderError(#[source] ParameterLoaderError),+ #[error("QLoRA linear layer not supported")]+ QLoRaNotSupported,+}++pub fn linear_block<const N: usize, B: Backend + 'static>(config: &LinearConfig,_has_biases: bool,input_dimension: usize,output_dimensions: [usize; N],- context: &MetalContext,- parameter_tree: &ParameterTree<MetalContext>,+ context: &B::Context,+ parameter_tree: &ParameterTree<B::Context>,input_array_id: ArrayId,output_array_id: ArrayId,-) -> Result<Box<dyn EncodableBlock<Metal>>, MetalError> {+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>+where+ B::Kernels: MatmulKernels,+{let output_dimension_sum: usize = output_dimensions.iter().sum();match config {LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {@@ -36,7 +59,7 @@ pub fn linear_block<const N: usize>(input_array_id,output_array_id,)- .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;+ .map_err(LayerError::QuantizedLinearError)?;Ok(Box::new(block))},LinearConfig::FullPrecision {@@ -51,23 +74,26 @@ pub fn linear_block<const N: usize>(input_array_id,output_array_id,)- .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;+ .map_err(LayerError::FullPrecisionLinearError)?;Ok(Box::new(block))},LinearConfig::QLoRA {..- } => Err(MetalError::Generic("QLoRA linear layer not supported".to_string())),+ } => Err(LayerError::QLoRaNotSupported),}}/// Creates an MLP block using the unfused implementation (separate up, activation, down)-pub fn mlp_block(+pub fn mlp_block<B: Backend + 'static>(config: &MLPConfig,model_dimension: usize,hidden_dimension: usize,- context: &MetalContext,- parameter_tree: &ParameterTree<MetalContext>,-) -> Result<Box<dyn EncodableBlock<Metal>>, MetalError> {+ context: &B::Context,+ parameter_tree: &ParameterTree<B::Context>,+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>+where+ B::Kernels: MatmulKernels,+{if let crate::config::MLPConfig::Dense(dense_config) = config {let data_type: DataType = dense_config.linear_config.activation_precision().into();@@ -78,14 +104,15 @@ pub fn mlp_block(model_dimension,[2 * hidden_dimension],context,- ¶meter_tree.subtree("up_projection").map_err(|error| MetalError::Generic(format!("{:?}", error)))?,+ ¶meter_tree.subtree("up_projection").map_err(LayerError::ParameterLoaderError)?,ArrayId::Main,ArrayId::MlpFusedUp,)?;// Gate activation + multiplylet gate_activation =- MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)?;+ MlpGateActMulEncodable::new(context, data_type, dense_config.activation.clone(), hidden_dimension)+ .map_err(LayerError::BackendError)?;// Down projectionlet down_projection = linear_block(@@ -94,7 +121,7 @@ pub fn mlp_block(hidden_dimension,[model_dimension],context,- ¶meter_tree.subtree("down_projection").map_err(|error| MetalError::Generic(format!("{:?}", error)))?,+ ¶meter_tree.subtree("down_projection").map_err(LayerError::ParameterLoaderError)?,ArrayId::MlpHidden,ArrayId::Main,)?;@@ -105,18 +132,18 @@ pub fn mlp_block(if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {let mixture_of_experts_block =MoeBlock::new(context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)- .map_err(|error| MetalError::Generic(format!("{:?}", error)))?;+ .map_err(LayerError::MoeBlockError)?;return Ok(Box::new(mixture_of_experts_block));}unreachable!("Unknown MLP config")}-pub fn embed_block(+pub fn embed_block<B: Backend + 'static>(config: &DecoderConfig,- context: &MetalContext,- parameter_tree: &ParameterTree<MetalContext>,-) -> Box<dyn EncodableBlock<Metal>> {+ context: &B::Context,+ parameter_tree: &ParameterTree<B::Context>,+) -> Box<dyn EncodableBlock<B>> {match &config.embedding_config {EmbeddingConfig::Tied {common,@@ -252,11 +279,14 @@ pub fn embed_block(}}-pub fn readout_block(+pub fn readout_block<B: Backend + 'static>(config: &DecoderConfig,- context: &MetalContext,- parameter_tree: &ParameterTree<MetalContext>,-) -> Box<dyn EncodableBlock<Metal>> {+ context: &B::Context,+ parameter_tree: &ParameterTree<B::Context>,+) -> Box<dyn EncodableBlock<B>>+where+ B::Kernels: MatmulKernels,+{match &config.embedding_config {EmbeddingConfig::Tied {precision,