A high-performance inference engine for AI models
more generic kernel wrappers and encodable blocks (#199)
uuuvn7f9ef618f85e
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex db22f269a..13ad56878 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -12,4 +12,7 @@ pub trait CommandBuffer {&self,callback: impl FnOnce(&<Self::Backend as Backend>::CopyEncoder) -> T,) -> T;++ fn submit(&self);+ fn wait_until_completed(&self);}diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/common/kernel/attention.rssimilarity index 67%rename from crates/uzu/src/backends/metal/kernel/attention/mod.rsrename to crates/uzu/src/backends/common/kernel/attention.rsindex 2f2f5cc67..76fd12630 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/common/kernel/attention.rs@@ -3,29 +3,24 @@ use std::{collections::{HashMap, hash_map::Entry},};-use metal::{MTLBuffer, MTLComputeCommandEncoder};-use objc2::{Message, runtime::ProtocolObject};-use crate::{DataType,- backends::{- common::{- gpu_types::{AttnMaskParams, AttnParams},- kernel::AttentionGemmKernel,- },- metal::{MetalContext, MetalError, kernel::dsl::AttentionGemmMetalKernel},+ backends::common::{+ Backend, Kernels,+ gpu_types::{AttnMaskParams, AttnParams},+ kernel::AttentionGemmKernel,},};const BQ: usize = 32;-pub struct AttentionGemmArguments<'a> {- pub queries_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(0)- pub keys_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(1)- pub values_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(2)- pub output_buffer: &'a ProtocolObject<dyn MTLBuffer>, // buffer(3)- pub mask_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(6)- pub sinks_buffer: Option<&'a ProtocolObject<dyn MTLBuffer>>, // buffer(7)+pub struct AttentionGemmArguments<'a, B: Backend> {+ pub queries_buffer: &'a B::NativeBuffer, // buffer(0)+ pub keys_buffer: &'a B::NativeBuffer, // buffer(1)+ pub values_buffer: &'a B::NativeBuffer, // buffer(2)+ pub output_buffer: &'a B::NativeBuffer, // buffer(3)+ pub mask_buffer: Option<&'a B::NativeBuffer>, // buffer(6)+ pub sinks_buffer: Option<&'a B::NativeBuffer>, // buffer(7)pub num_heads: usize,pub num_groups: usize,pub suffix_length: usize, // qL@@ -37,12 +32,12 @@ pub struct AttentionGemmArguments<'a> {pub scale: f32,}-pub struct AttentionGemmBlock {+pub struct AttentionGemmBlock<B: Backend> {data_type: DataType,- cache: RefCell<HashMap<KernelKey, AttentionGemmMetalKernel>>,+ cache: RefCell<HashMap<KernelKey, <B::Kernels as Kernels>::AttentionGemmKernel>>,}-impl AttentionGemmBlock {+impl<B: Backend> AttentionGemmBlock<B> {pub fn new(data_type: DataType) -> Self {let cache = RefCell::new(HashMap::new());Self {@@ -53,10 +48,10 @@ impl AttentionGemmBlock {pub fn encode(&self,- context: &MetalContext,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: &AttentionGemmArguments,- ) -> Result<(), MetalError> {+ context: &B::Context,+ compute_encoder: &B::ComputeEncoder,+ args: &AttentionGemmArguments<B>,+ ) -> Result<(), B::Error> {let bk: usize = if args.head_dim < 128 {32} else {@@ -80,7 +75,7 @@ impl AttentionGemmBlock {let kernel = match map.entry(key) {Entry::Occupied(entry) => entry.into_mut(),Entry::Vacant(entry) => {- let kernel = AttentionGemmMetalKernel::new(+ let kernel = <B::Kernels as Kernels>::AttentionGemmKernel::new(context,self.data_type,bk as u32,@@ -130,22 +125,17 @@ impl AttentionGemmBlock {// We use a shared bias matrix for all heads/batches.m_strides: [0, 0, args.sequence_length as i64],};- let attn_mask_params_opt = args.mask_buffer.map(|_| std::slice::from_ref(&attn_mask_params));-- let mask_buffer_retain = args.mask_buffer.map(|b| b.retain());- let mask_buffer_opt = mask_buffer_retain.as_ref().map(|b| b);- let sinks_buffer_retain = args.sinks_buffer.map(|b| b.retain());- let sinks_buffer_opt = sinks_buffer_retain.as_ref().map(|b| b);+ let attn_mask_params_opt = args.mask_buffer.map(|_| attn_mask_params);kernel.encode(- &args.queries_buffer.retain(),- &args.keys_buffer.retain(),- &args.values_buffer.retain(),- &args.output_buffer.retain(),- std::slice::from_ref(¶ms),+ args.queries_buffer,+ args.keys_buffer,+ args.values_buffer,+ args.output_buffer,+ params,attn_mask_params_opt,- mask_buffer_opt,- sinks_buffer_opt,+ args.mask_buffer,+ args.sinks_buffer,args.num_heads as u32,args.suffix_length as u32,&compute_encoder,diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rsnew file mode 100644index 000000000..7139b277c--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -0,0 +1,47 @@+use crate::{+ DataType,+ backends::common::{Backend, Kernels, kernel::MlpGateActMulKernel},+ config::Activation,+};++pub struct MlpGateActMulEncodable<B: Backend> {+ kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,+ activation: Activation,+ hidden_dim: usize,+}++impl<B: Backend> MlpGateActMulEncodable<B> {+ pub fn new(+ context: &B::Context,+ data_type: DataType,+ activation: Activation,+ hidden_dim: usize,+ ) -> Result<Self, B::Error> {+ let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type)?;+ Ok(Self {+ kernel,+ activation,+ hidden_dim,+ })+ }++ pub fn encode(+ &self,+ encoder: &B::ComputeEncoder,+ fused_up: &B::NativeBuffer,+ hidden: &B::NativeBuffer,+ m: i32,+ ) -> Result<(), B::Error> {+ let act_type = match self.activation {+ Activation::SiLU {+ ..+ } => 0,+ Activation::Gelu => 1,+ Activation::Identity => {+ panic!("Identity activation is not supported for kernel")+ },+ };+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rsindex ca042ab4e..1ff1f73f7 100644--- a/crates/uzu/src/backends/common/kernel/mod.rs+++ b/crates/uzu/src/backends/common/kernel/mod.rs@@ -2,6 +2,9 @@include!(concat!(env!("OUT_DIR"), "/traits.rs"));+pub mod attention;pub mod kv_cache_update;pub mod matmul;+pub mod mlp_gate_act_mul;pub mod sampling;+pub mod ssd_prefill;diff --git a/crates/uzu/src/backends/common/kernel/ssd_prefill.rs b/crates/uzu/src/backends/common/kernel/ssd_prefill.rsnew file mode 100644index 000000000..b65adaa73--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/ssd_prefill.rs@@ -0,0 +1,134 @@+use crate::{+ DataType,+ backends::common::{+ Backend, Kernels,+ kernel::{SSDPrefill64Kernel, SSDPrefillKernel, SSDPrefillSequentialKernel},+ },+};++#[derive(Copy, Clone, Debug, PartialEq, Eq)]+pub enum SSDPrefillMode {+ Sequential,+ SinglePass,+}++pub struct SSDPrefillArguments<'a, B: Backend> {+ pub x: &'a B::NativeBuffer,+ pub dt: &'a B::NativeBuffer, // raw dt values+ pub b: &'a B::NativeBuffer,+ pub c: &'a B::NativeBuffer,+ pub d: &'a B::NativeBuffer,+ pub z: &'a B::NativeBuffer,+ pub state: &'a B::NativeBuffer,+ pub y: &'a B::NativeBuffer,+ pub suffix_len: usize,+ pub group_size: i32,+ pub state_size: i32,+ pub x_strides: [usize; 3],+ pub dt_strides: [usize; 2],+ pub cb_strides: [usize; 3],+ pub state_strides: [usize; 3],+ pub channels: usize,+ pub head_dim: usize,+}++pub struct SSDPrefillKernels<B: Backend> {+ single: <B::Kernels as Kernels>::SSDPrefillKernel,+ single_64: <B::Kernels as Kernels>::SSDPrefill64Kernel,+ sequential: <B::Kernels as Kernels>::SSDPrefillSequentialKernel,+}++impl<B: Backend> SSDPrefillKernels<B> {+ pub fn new(+ context: &B::Context,+ data_type: DataType,+ ) -> Result<Self, B::Error> {+ let single = <B::Kernels as Kernels>::SSDPrefillKernel::new(context, data_type)?;+ let single_64 = <B::Kernels as Kernels>::SSDPrefill64Kernel::new(context, data_type)?;+ let sequential = <B::Kernels as Kernels>::SSDPrefillSequentialKernel::new(context, data_type)?;+ Ok(Self {+ single,+ single_64,+ sequential,+ })+ }++ pub fn encode(+ &self,+ compute_encoder: &B::ComputeEncoder,+ args: SSDPrefillArguments<B>,+ mode: SSDPrefillMode,+ ) {+ let x_strides: Vec<u32> = args.x_strides.iter().map(|x| *x as u32).collect();+ let dt_strides: Vec<u32> = args.dt_strides.iter().map(|x| *x as u32).collect();+ let cb_strides: Vec<u32> = args.cb_strides.iter().map(|x| *x as u32).collect();+ let state_strides: Vec<u32> = args.state_strides.iter().map(|x| *x as u32).collect();++ if mode == SSDPrefillMode::SinglePass {+ if args.state_size == 64 {+ self.single_64.encode(+ args.x,+ args.dt,+ args.b,+ args.c,+ args.d,+ args.z,+ args.state,+ args.y,+ args.suffix_len as u32,+ args.group_size,+ args.state_size,+ x_strides.as_slice(),+ dt_strides.as_slice(),+ cb_strides.as_slice(),+ state_strides.as_slice(),+ args.channels as u32,+ args.head_dim as u32,+ compute_encoder,+ )+ } else {+ self.single.encode(+ args.x,+ args.dt,+ args.b,+ args.c,+ args.d,+ args.z,+ args.state,+ args.y,+ args.suffix_len as u32,+ args.group_size,+ args.state_size,+ x_strides.as_slice(),+ dt_strides.as_slice(),+ cb_strides.as_slice(),+ state_strides.as_slice(),+ args.channels as u32,+ args.head_dim as u32,+ compute_encoder,+ )+ }+ } else if mode == SSDPrefillMode::Sequential {+ self.sequential.encode(+ args.x,+ args.dt,+ args.b,+ args.c,+ args.d,+ args.z,+ args.state,+ args.y,+ args.suffix_len as u32,+ args.group_size,+ args.state_size,+ x_strides.as_slice(),+ dt_strides.as_slice(),+ cb_strides.as_slice(),+ state_strides.as_slice(),+ args.channels as u32,+ args.head_dim as u32,+ compute_encoder,+ )+ }+ }+}diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex 296a1ea3f..b300da23d 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -6,7 +6,7 @@ use objc2::{rc::Retained, runtime::ProtocolObject};use super::{MetalContext,encodable_block::{- ClassifierLayer, ClassifierPredictionHead,+ ClassifierLayer,transformer_layer::{embed_block, linear_block},},kernel::dsl::SigmoidMetalKernel,@@ -18,7 +18,7 @@ use crate::{metal::{Metal, error::ClassifierError},},config::{ClassifierModelConfig, ModelMetadata},- encodable_block::{Activation, EncodableBlock, Normalization, Pooling, Rope},+ encodable_block::{Activation, ClassifierPredictionHead, EncodableBlock, Normalization, Pooling, Rope},forward_pass::{model_shape::ModelShape,scratch_buffers::ScratchBuffers,@@ -40,7 +40,7 @@ pub struct ClassifierContext {pub embed: Box<dyn EncodableBlock<Metal>>,pub embedding_norm: Normalization<Metal>,- pub layers: Box<[ClassifierLayer]>,+ pub layers: Box<[ClassifierLayer<Metal>]>,pub output_norm: Normalization<Metal>,pub global_rope: Rc<Box<dyn EncodableBlock<Metal>>>,pub local_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex 904dbdbe4..5523cea83 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -1,3 +1,5 @@+use std::ops::Deref;+use metal::{MTLCommandBuffer, MTLCommandEncoder};use objc2::{rc::Retained, runtime::ProtocolObject};@@ -32,4 +34,12 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {ret}++ fn submit(&self) {+ self.commit();+ }++ fn wait_until_completed(&self) {+ self.deref().wait_until_completed();+ }}diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsindex 7684e4728..290f445d8 100644--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs@@ -1,43 +1,44 @@use std::rc::Rc;-#[cfg(not(feature = "tracing"))]-use metal::MTLCommandEncoder;-use metal::{MTLCommandBuffer, MTLComputeCommandEncoder};-use objc2::{- rc::{Retained, autoreleasepool},- runtime::ProtocolObject,-};+use objc2::rc::autoreleasepool;-use super::{Attention, transformer_layer};+use super::transformer_layer;+#[cfg(not(feature = "tracing"))]+use crate::backends::common::CommandBuffer;use crate::{DataType,- backends::metal::{Metal, MetalContext},+ backends::{+ common::Backend,+ metal::{Metal, MetalContext},+ },config::TransformerLayerConfig,- encodable_block::{EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy},+ encodable_block::{+ Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,+ },forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};-pub struct ClassifierLayer {+pub struct ClassifierLayer<B: Backend> {#[cfg_attr(not(feature = "tracing"), allow(dead_code))]layer_index: usize,- copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>>,- pre_attention_norm: Option<Box<dyn EncodableBlock<Metal>>>,- qkv_projection: Box<dyn EncodableBlock<Metal>>,- qk_norm: Option<Box<dyn EncodableBlock<Metal>>>,- rope: Rc<Box<dyn EncodableBlock<Metal>>>,- attention: Box<dyn EncodableBlock<Metal>>,- out_projection: Box<dyn EncodableBlock<Metal>>,- post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>>,- mixer_residual_add: Box<dyn EncodableBlock<Metal>>,- copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<Metal>>,- pre_mlp_norm: Box<dyn EncodableBlock<Metal>>,- mlp: Box<dyn EncodableBlock<Metal>>,- post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>>,- mlp_residual_add: Box<dyn EncodableBlock<Metal>>,+ copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<B>>,+ pre_attention_norm: Option<Box<dyn EncodableBlock<B>>>,+ qkv_projection: Box<dyn EncodableBlock<B>>,+ qk_norm: Option<Box<dyn EncodableBlock<B>>>,+ rope: Rc<Box<dyn EncodableBlock<B>>>,+ attention: Box<dyn EncodableBlock<B>>,+ out_projection: Box<dyn EncodableBlock<B>>,+ post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,+ mixer_residual_add: Box<dyn EncodableBlock<B>>,+ copy_main_to_shortcut_mlp: Box<dyn EncodableBlock<B>>,+ pre_mlp_norm: Box<dyn EncodableBlock<B>>,+ mlp: Box<dyn EncodableBlock<B>>,+ post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,+ mlp_residual_add: Box<dyn EncodableBlock<B>>,}-impl ClassifierLayer {+impl ClassifierLayer<Metal> {pub fn new(mtl_context: Rc<MetalContext>,layer_config: &TransformerLayerConfig,@@ -247,20 +248,18 @@ impl ClassifierLayer {}}-impl EncodableBlock<Metal> for ClassifierLayer {+impl<B: Backend> EncodableBlock<B> for ClassifierLayer<B> {fn encode(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {#[cfg(not(feature = "tracing"))]{if self.supports_shared_encoder() {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &encoder);- encoder.end_encoding();+ command_buffer+ .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));return;}}@@ -384,9 +383,9 @@ impl EncodableBlock<Metal> for ClassifierLayer {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,) {debug_assert!(self.supports_shared_encoder(),diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rsindex 03ddee61b..ff065fb21 100644--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs@@ -2,15 +2,15 @@use std::rc::Rc;-use metal::{MTLCommandBuffer, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};-use super::LayerExecutables;use crate::{DataType, DecoderConfig,- backends::metal::{- Metal, MetalContext,- encodable_block::transformer_layer::{embed_block, readout_block},+ backends::{+ common::Backend,+ metal::{+ Metal, MetalContext,+ encodable_block::transformer_layer::{embed_block, readout_block},+ },},config::{DecoderLayerType, MixerConfig},encodable_block::{EncodableBlock, EncodingParameters, RMSNorm, Rope},@@ -22,16 +22,16 @@ use crate::{};/// Full decoder executable with all layers and components.-pub struct Decoder {- pub embed: Box<dyn EncodableBlock<Metal>>,- pub layers: Box<[LayerExecutables]>,- pub norm: Box<dyn EncodableBlock<Metal>>,- pub readout: Box<dyn EncodableBlock<Metal>>,- pub global_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,- pub local_rope: Option<Rc<Box<dyn EncodableBlock<Metal>>>>,+pub struct Decoder<B: Backend> {+ pub embed: Box<dyn EncodableBlock<B>>,+ pub layers: Box<[LayerExecutables<B>]>,+ pub norm: Box<dyn EncodableBlock<B>>,+ pub readout: Box<dyn EncodableBlock<B>>,+ pub global_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,+ pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,}-impl Decoder {+impl Decoder<Metal> {pub fn new(mtl_context: Rc<MetalContext>,decoder_config: Rc<DecoderConfig>,@@ -167,21 +167,21 @@ impl Decoder {}}-impl EncodableBlock<Metal> for Decoder {+impl<B: Backend> EncodableBlock<B> for Decoder<B> {fn encode_with_shared_encoder(&self,- _state: &mut ForwardPassState<Metal>,- _parameters: &EncodingParameters<Metal>,- _encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ _state: &mut ForwardPassState<B>,+ _parameters: &EncodingParameters<B>,+ _encoder: &B::ComputeEncoder,) {unimplemented!("Decoder does not support shared encoder")}fn encode(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {self.embed.encode(state, parameters, command_buffer);diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsindex a847f842c..ff7922f69 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -2,41 +2,40 @@use std::rc::Rc;-#[cfg(not(feature = "tracing"))]-use metal::MTLCommandEncoder;-use metal::{MTLCommandBuffer, MTLComputeCommandEncoder};-use objc2::{- rc::{Retained, autoreleasepool},- runtime::ProtocolObject,-};+use objc2::rc::autoreleasepool;-use super::{- super::{Attention, MambaMixer, ShortConvMixer, transformer_layer},- MixerExecutables,-};+use super::{super::transformer_layer, MixerExecutables};+#[cfg(not(feature = "tracing"))]+use crate::backends::common::CommandBuffer;use crate::{DataType, DecoderLayerConfig,- backends::metal::{Metal, MetalContext},+ backends::{+ common::Backend,+ metal::{Metal, MetalContext},+ },config::{DecoderLayerType, MixerConfig},- encodable_block::{EncodableBlock, EncodingParameters, QKNorm, RMSNorm, TensorAddSwap, TensorCopy},+ encodable_block::{+ Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,+ TensorCopy,+ },forward_pass::state::{ArrayId, ForwardPassState},parameters::ParameterTree,};/// A single decoder layer with all its components.-pub struct LayerExecutables {+pub struct LayerExecutables<B: Backend> {pub layer_index: usize,- pub copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>>,- pub pre_attention_norm: Box<dyn EncodableBlock<Metal>>,- pub(crate) mixer: MixerExecutables,- pub post_attention_norm: Option<Box<dyn EncodableBlock<Metal>>>,- pub main_shortcut_add_swap: Box<dyn EncodableBlock<Metal>>,- pub pre_mlp_norm: Box<dyn EncodableBlock<Metal>>,- pub mlp: Box<dyn EncodableBlock<Metal>>,- pub post_mlp_norm: Option<Box<dyn EncodableBlock<Metal>>>,+ pub copy_main_to_shortcut: Box<dyn EncodableBlock<B>>,+ pub pre_attention_norm: Box<dyn EncodableBlock<B>>,+ pub(crate) mixer: MixerExecutables<B>,+ pub post_attention_norm: Option<Box<dyn EncodableBlock<B>>>,+ pub main_shortcut_add_swap: Box<dyn EncodableBlock<B>>,+ pub pre_mlp_norm: Box<dyn EncodableBlock<B>>,+ pub mlp: Box<dyn EncodableBlock<B>>,+ pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,}-impl LayerExecutables {+impl LayerExecutables<Metal> {#[allow(clippy::too_many_arguments)]pub fn new(mtl_context: Rc<MetalContext>,@@ -259,22 +258,20 @@ impl LayerExecutables {}}-impl EncodableBlock<Metal> for LayerExecutables {+impl<B: Backend> EncodableBlock<B> for LayerExecutables<B> {fn encode(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {// In non-tracing builds, if every sub-block supports shared encoding,// we can run the entire layer in a single compute encoder.#[cfg(not(feature = "tracing"))]{if self.supports_shared_encoder() {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &encoder);- encoder.end_encoding();+ command_buffer+ .with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));return;}}@@ -432,9 +429,9 @@ impl EncodableBlock<Metal> for LayerExecutables {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,) {debug_assert!(self.supports_shared_encoder(),diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rsindex 0ac31a7b4..2ac32fc5d 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/mixer.rs@@ -2,21 +2,21 @@use std::rc::Rc;-use crate::{backends::metal::Metal, encodable_block::EncodableBlock};+use crate::{backends::common::Backend, encodable_block::EncodableBlock};/// Mixer component - either attention, state space, or short conv.-pub(crate) enum MixerExecutables {+pub(crate) enum MixerExecutables<B: Backend> {Attention {- qkv_projection: Box<dyn EncodableBlock<Metal>>,- qk_norm: Option<Box<dyn EncodableBlock<Metal>>>,- rope: Rc<Box<dyn EncodableBlock<Metal>>>,- attention: Box<dyn EncodableBlock<Metal>>,- out_projection: Box<dyn EncodableBlock<Metal>>,+ qkv_projection: Box<dyn EncodableBlock<B>>,+ qk_norm: Option<Box<dyn EncodableBlock<B>>>,+ rope: Rc<Box<dyn EncodableBlock<B>>>,+ attention: Box<dyn EncodableBlock<B>>,+ out_projection: Box<dyn EncodableBlock<B>>,},StateSpace {- mixer: Box<dyn EncodableBlock<Metal>>,+ mixer: Box<dyn EncodableBlock<B>>,},ShortConv {- mixer: Box<dyn EncodableBlock<Metal>>,+ mixer: Box<dyn EncodableBlock<B>>,},}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 7c704bc60..ebcf7ef7d 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -6,20 +6,14 @@//! Encodables implement `EncodableBlock` and orchestrate one or more//! kernels to perform operations on `ForwardPassState`.-mod attention;mod classifier_layer;mod decoder;mod embedding;mod layer;mod linear;-mod mamba_mixer;-mod mlp;mod moe_block;-mod prediction_head;-mod short_conv_mixer;pub mod transformer_layer;-pub use attention::Attention;pub use classifier_layer::ClassifierLayer;pub use decoder::Decoder;pub use embedding::{@@ -28,8 +22,4 @@ pub use embedding::{};pub use layer::LayerExecutables;pub use linear::{FullPrecisionLinear, QuantizedLinear};-pub(crate) use mamba_mixer::MambaMixer;-pub use mlp::MlpBlock;pub use moe_block::{MoeBlock, SharedMoeWeights};-pub use prediction_head::ClassifierPredictionHead;-pub(crate) use short_conv_mixer::ShortConvMixer;diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rsdeleted file mode 100644index 71154dbc1..000000000--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs+++ /dev/null@@ -1,106 +0,0 @@-//! Prediction head encodable for classification output.--#[cfg(feature = "tracing")]-use metal::MTLBlitCommandEncoder;-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};--#[cfg(feature = "tracing")]-use crate::forward_pass::state::ArrayId;-use crate::{- backends::metal::Metal,- encodable_block::{EncodableBlock, EncodingParameters},- forward_pass::state::ForwardPassState,-};--pub struct ClassifierPredictionHead {- dense: Box<dyn EncodableBlock<Metal>>,- activation: Box<dyn EncodableBlock<Metal>>,- norm: Box<dyn EncodableBlock<Metal>>,- readout: Box<dyn EncodableBlock<Metal>>,- #[cfg_attr(not(feature = "tracing"), allow(dead_code))]- num_labels: usize,-}--impl ClassifierPredictionHead {- pub fn new(- dense: Box<dyn EncodableBlock<Metal>>,- activation: Box<dyn EncodableBlock<Metal>>,- norm: Box<dyn EncodableBlock<Metal>>,- readout: Box<dyn EncodableBlock<Metal>>,- num_labels: usize,- ) -> Self {- Self {- dense,- activation,- norm,- readout,- num_labels,- }- }-}--impl EncodableBlock<Metal> for ClassifierPredictionHead {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- if self.supports_shared_encoder() {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &encoder);- encoder.end_encoding();- } else {- self.dense.encode(state, parameters, command_buffer);- self.activation.encode(state, parameters, command_buffer);- self.norm.encode(state, parameters, command_buffer);- self.readout.encode(state, parameters, command_buffer);- }-- #[cfg(feature = "tracing")]- {- let traces_rc = state.traces().clone();- let logits_arrays = state.arrays(&[ArrayId::ClassifierPredictionHeadLogits]);- let logits_array_ref = logits_arrays[0].borrow();- let linear_output_buffer = logits_array_ref.buffer();- let data_type = logits_array_ref.data_type();- let batch_size = logits_array_ref.shape()[0];-- let traces_ref = traces_rc.borrow();- let trace_logits = traces_ref.logits.borrow();- let dst_trace_buf = trace_logits.buffer();-- let copy_size_bytes = batch_size * self.num_labels * data_type.size_in_bytes();-- let blit = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");- blit.copy_buffer_to_buffer(linear_output_buffer, 0, dst_trace_buf, 0, copy_size_bytes);- blit.end_encoding();- }-- if parameters.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- self.dense.supports_shared_encoder()- && self.activation.supports_shared_encoder()- && self.norm.supports_shared_encoder()- && self.readout.supports_shared_encoder()- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) {- self.dense.encode_with_shared_encoder(state, parameters, encoder);- self.activation.encode_with_shared_encoder(state, parameters, encoder);- self.norm.encode_with_shared_encoder(state, parameters, encoder);- self.readout.encode_with_shared_encoder(state, parameters, encoder);- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 8bee2f346..dcc4987c2 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -1,12 +1,15 @@use super::{- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock, MoeBlock,+ FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, MoeBlock,QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,- backends::metal::{Metal, MetalContext, MetalError, kernel::mlp::MlpGateActMulEncodable},+ backends::{+ common::kernel::mlp_gate_act_mul::MlpGateActMulEncodable,+ metal::{Metal, MetalContext, MetalError},+ },config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},- encodable_block::EncodableBlock,+ encodable_block::{EncodableBlock, MlpBlock},forward_pass::state::ArrayId,parameters::ParameterTree,};diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 7f9dd6c63..06ed9897c 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -45,8 +45,8 @@ KERNEL(AttentionGemm)(const device T* k,const device T* v,device T* o,- const constant AttnParams* params,- const constant AttnMaskParams* mask_params OPTIONAL(has_mask),+ const constant AttnParams& params,+ const constant AttnMaskParams& mask_params OPTIONAL(has_mask),const device T* mask OPTIONAL(has_mask),const device float* sinks OPTIONAL(has_sinks),const constant uint& num_heads,@@ -78,21 +78,21 @@ KERNEL(AttentionGemm)(const uint head_idx = tgid_y;const uint q_tile_idx = tgid_x;- q += batch_idx * params->q_strides[0] + head_idx * params->q_strides[1] +- q_tile_idx * int64_t(BQ) * params->q_strides[2];+ q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] ++ q_tile_idx * int64_t(BQ) * params.q_strides[2];- const int kv_head_idx = int(tgid_y) / params->gqa_factor;- k += batch_idx * params->k_strides[0] +- int64_t(kv_head_idx) * params->k_strides[1];- v += batch_idx * params->v_strides[0] +- int64_t(kv_head_idx) * params->v_strides[1];+ const int kv_head_idx = int(tgid_y) / params.gqa_factor;+ k += batch_idx * params.k_strides[0] ++ int64_t(kv_head_idx) * params.k_strides[1];+ v += batch_idx * params.v_strides[0] ++ int64_t(kv_head_idx) * params.v_strides[1];- o += batch_idx * params->o_strides[0] + head_idx * params->o_strides[1] +- q_tile_idx * int64_t(BQ) * params->o_strides[2];+ o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] ++ q_tile_idx * int64_t(BQ) * params.o_strides[2];if (has_mask) {- mask += batch_idx * mask_params->m_strides[0] +- head_idx * mask_params->m_strides[1];+ mask += batch_idx * mask_params.m_strides[0] ++ head_idx * mask_params.m_strides[1];}// -------------------------------------------------------------------------@@ -105,10 +105,6 @@ KERNEL(AttentionGemm)(constexpr short LDK_tgp = BD + padK; // K stored as [BK, BD] row-majorconstexpr short LDV_tgp = BD + padV;- constexpr short tgp_mem_k = BK * (BD + padK);- constexpr short tgp_mem_v = BK * (BD + padV);- constexpr short tgp_mem_s = tgp_mem_k > tgp_mem_v ? tgp_mem_k : tgp_mem_v;-threadgroup T* Qs = q_smem;threadgroup T* Ks = kv_smem;threadgroup T* Vs = kv_smem;@@ -140,15 +136,15 @@ KERNEL(AttentionGemm)(/*reduction_dim=*/0,/*tgp_size=*/WM * WN * 32>;- const int q_src_ld = int(params->q_strides[2]);- const int k_src_ld = int(params->k_strides[2]);- const int v_src_ld = int(params->v_strides[2]);+ const int q_src_ld = int(params.q_strides[2]);+ const int k_src_ld = int(params.k_strides[2]);+ const int v_src_ld = int(params.v_strides[2]);thread QBlockLoader loader_q(q, q_src_ld, Qs, simd_group_id, simd_lane_id);thread KBlockLoader loader_k(k, k_src_ld, Ks, simd_group_id, simd_lane_id);thread VBlockLoader loader_v(v, v_src_ld, Vs, simd_group_id, simd_lane_id);- TransformScale<T> ts(static_cast<T>(params->scale * M_LOG2E_F));+ TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));// -------------------------------------------------------------------------// MMA tiles@@ -204,8 +200,8 @@ KERNEL(AttentionGemm)(// Load Q block once (and apply scaling)threadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_q && int(tgid_x) == params->nq_aligned) {- loader_q.load_safe(short2(BD, params->q_rem));+ if (!align_q && int(tgid_x) == params.nq_aligned) {+ loader_q.load_safe(short2(BD, params.q_rem));} else {loader_q.load_unsafe();}@@ -223,22 +219,22 @@ KERNEL(AttentionGemm)(}// Determine K block loop limit (causal can early-stop)- int kb_lim = params->nk;+ int kb_lim = params.nk;if (do_causal) {- const int q_max = (int(tgid_x) + 1) * BQ + params->q_off;+ const int q_max = (int(tgid_x) + 1) * BQ + params.q_off;kb_lim = (q_max + BK - 1) / BK;- kb_lim = min(params->nk, kb_lim);+ kb_lim = min(params.nk, kb_lim);}const int q_rel = int(tgid_x) * BQ + int(tm) + int(sm); // [0, q_len)- const int q_abs = q_rel + params->q_off; // [0, k_len)+ const int q_abs = q_rel + params.q_off; // [0, k_len)// Loop over KV blocksfor (int kb = 0; kb < kb_lim; kb++) {// Load K blockthreadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_k && kb == params->nk_aligned) {- loader_k.load_safe(short2(BD, params->k_rem));+ if (!align_k && kb == params.nk_aligned) {+ loader_k.load_safe(short2(BD, params.k_rem));} else {loader_k.load_unsafe();}@@ -264,8 +260,8 @@ KERNEL(AttentionGemm)(}// Mask out tail keys for the last (unaligned) K block- if (!align_k && kb == params->nk_aligned) {- const int k_rem = params->k_rem;+ if (!align_k && kb == params.nk_aligned) {+ const int k_rem = params.k_rem;UZU_PRAGMA_UNROLLfor (short j = 0; j < TK; j++) {thread auto& frag = Stile.frag_at(0, j);@@ -299,8 +295,8 @@ KERNEL(AttentionGemm)(}// Add external mask (additive bias in natural-log domain; convert to log2)- if (has_mask && q_rel < params->q_len) {- const int64_t row_stride = mask_params->m_strides[2];+ if (has_mask && q_rel < params.q_len) {+ const int64_t row_stride = mask_params.m_strides[2];const int64_t row_base = int64_t(q_rel) * row_stride;UZU_PRAGMA_UNROLL@@ -311,12 +307,12 @@ KERNEL(AttentionGemm)(const int k0 = col_base;const int k1 = col_base + 1;- if (k0 < params->k_len) {+ if (k0 < params.k_len) {AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k0)]);mv = metal::max(mv, static_cast<AccumType>(-1e9f));frag[0] += M_LOG2E_F * mv;}- if (k1 < params->k_len) {+ if (k1 < params.k_len) {AccumType mv = static_cast<AccumType>(mask[row_base + int64_t(k1)]);mv = metal::max(mv, static_cast<AccumType>(-1e9f));frag[1] += M_LOG2E_F * mv;@@ -326,8 +322,8 @@ KERNEL(AttentionGemm)(// Load V block (overwriting K in shared memory)threadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_k && kb == params->nk_aligned) {- loader_v.load_safe(short2(BD, params->k_rem));+ if (!align_k && kb == params.nk_aligned) {+ loader_v.load_safe(short2(BD, params.k_rem));} else {loader_v.load_unsafe();}@@ -415,11 +411,11 @@ KERNEL(AttentionGemm)(threadgroup_barrier(mem_flags::mem_none);- // Store results (O is row-major with row-stride params->o_strides[2])- o += int64_t(tm + sm) * params->o_strides[2] + int64_t(sn);+ // Store results (O is row-major with row-stride params.o_strides[2])+ o += int64_t(tm + sm) * params.o_strides[2] + int64_t(sn);- if (!align_q && int(tgid_x) == params->nq_aligned) {- const short2 dst_tile_dims = short2(BD - sn, params->q_rem - (tm + sm));+ if (!align_q && int(tgid_x) == params.nq_aligned) {+ const short2 dst_tile_dims = short2(BD - sn, params.q_rem - (tm + sm));if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {return;@@ -427,10 +423,10 @@ KERNEL(AttentionGemm)(Otile.template store_safe<T, 1, 1>(o,- int(params->o_strides[2]),+ int(params.o_strides[2]),dst_tile_dims);} else {- Otile.template store<T, 1, 1>(o, int(params->o_strides[2]));+ Otile.template store<T, 1, 1>(o, int(params.o_strides[2]));}-}\ No newline at end of file+}diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsdeleted file mode 100644index 550d5cca3..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs+++ /dev/null@@ -1,116 +0,0 @@-use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLFunctionConstantValues};-use objc2::{rc::Retained, runtime::ProtocolObject};--use crate::{- DataType,- backends::{- common::kernel::MlpGateActMulKernel,- metal::{FunctionConstantValuesSetValue, MetalContext, MetalError, kernel::dsl::MlpGateActMulMetalKernel},- },- config::Activation,-};--// MLP Fused Epilogue Function Constant Indices-// These must match the values in kernel/common/mlp_epilogue.h-pub const MLP_FUSED_FC_INDEX: u64 = 50;-pub const MLP_HIDDEN_DIM_FC_INDEX: u64 = 51;-pub const MLP_ACTIVATION_FC_INDEX: u64 = 52;--/// MLP activation type for fused kernels-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-#[repr(u32)]-pub enum MlpActivationType {- SiLU = 0,- Gelu = 1,-}--impl From<&Activation> for MlpActivationType {- fn from(act: &Activation) -> Self {- match act {- Activation::SiLU {- ..- } => MlpActivationType::SiLU,- Activation::Gelu => MlpActivationType::Gelu,- Activation::Identity => {- panic!("Identity activation not supported for MLP fusion")- },- }- }-}--/// Configuration for MLP fused matmul epilogue-#[derive(Debug, Clone, Copy)]-pub struct MlpFusedConfig {- pub hidden_dim: u32,- pub activation: MlpActivationType,-}--impl MlpFusedConfig {- pub fn new(- hidden_dim: usize,- activation: &Activation,- ) -> Self {- Self {- hidden_dim: hidden_dim as u32,- activation: MlpActivationType::from(activation),- }- }-- /// Create function constants for MLP fused matmul- pub fn make_function_constants(&self) -> Retained<MTLFunctionConstantValues> {- let fcv = MTLFunctionConstantValues::new();- fcv.set_value(&true, MLP_FUSED_FC_INDEX as usize);- fcv.set_value(&self.hidden_dim, MLP_HIDDEN_DIM_FC_INDEX as usize);- fcv.set_value(&(self.activation as u32), MLP_ACTIVATION_FC_INDEX as usize);- fcv- }-}--/// Create function constants for non-fused (standard) matmul-pub fn make_non_fused_function_constants() -> Retained<MTLFunctionConstantValues> {- let fcv = MTLFunctionConstantValues::new();- fcv.set_value(&false, MLP_FUSED_FC_INDEX as usize);- fcv-}--pub struct MlpGateActMulEncodable {- kernel: MlpGateActMulMetalKernel,- activation: Activation,- hidden_dim: usize,-}--impl MlpGateActMulEncodable {- pub fn new(- context: &MetalContext,- data_type: DataType,- activation: Activation,- hidden_dim: usize,- ) -> Result<Self, MetalError> {- let kernel = MlpGateActMulMetalKernel::new(context, data_type.into())?;- Ok(Self {- kernel,- activation,- hidden_dim,- })- }-- pub fn encode(- &self,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- fused_up: &Retained<ProtocolObject<dyn MTLBuffer>>,- hidden: &Retained<ProtocolObject<dyn MTLBuffer>>,- m: i32,- ) -> Result<(), MetalError> {- let act_type = match self.activation {- Activation::SiLU {- ..- } => 0,- Activation::Gelu => 1,- Activation::Identity => {- panic!("Identity activation is not supported for kernel")- },- };- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, encoder);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex e1400f79a..6f02adf2c 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,4 +1,3 @@-pub mod attention;use crate::backends::common::kernel::matmul::MatmulKernels;pub mod dsl {include!(concat!(env!("OUT_DIR"), "/dsl.rs"));@@ -6,14 +5,8 @@ pub mod dsl {pub(super) const MTLB: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));pub mod matmul;-pub mod mlp;-pub use mlp::{- MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX, MlpActivationType, MlpFusedConfig,- make_non_fused_function_constants,-};pub mod moe;pub mod quant_matmul;-pub mod ssm;pub use matmul::{MatmulArguments, MatmulKernel};pub use moe::{@@ -22,7 +15,6 @@ pub use moe::{MoeRouterTopKArguments, MoeRouterTopKKernel, MoeScatterArguments, MoeScatterError, MoeScatterKernels,MoeScatterWithMapArguments,};-pub use ssm::{SSDPrefillArguments, SSDPrefillMode};impl MatmulKernels for dsl::MetalKernels {type FullPrecisionMatmulKernel = MatmulKernel;diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rsdeleted file mode 100644index a87e87362..000000000--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs+++ /dev/null@@ -1,2 +0,0 @@-pub mod ssd_prefill;-pub use ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode};diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rsdeleted file mode 100644index d5a19dd69..000000000--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs+++ /dev/null@@ -1,135 +0,0 @@-use metal::{MTLBuffer, MTLComputeCommandEncoder};-use objc2::{Message, runtime::ProtocolObject};--use crate::{- DataType,- backends::{- common::kernel::{SSDPrefill64Kernel as _, SSDPrefillKernel as _, SSDPrefillSequentialKernel as _},- metal::{- MetalContext, MetalError,- kernel::dsl::{SSDPrefill64MetalKernel, SSDPrefillMetalKernel, SSDPrefillSequentialMetalKernel},- },- },-};--#[derive(Copy, Clone, Debug, PartialEq, Eq)]-pub enum SSDPrefillMode {- Sequential,- SinglePass,-}--pub struct SSDPrefillArguments<'a> {- pub x: &'a ProtocolObject<dyn MTLBuffer>,- pub dt: &'a ProtocolObject<dyn MTLBuffer>, // raw dt values- pub b: &'a ProtocolObject<dyn MTLBuffer>,- pub c: &'a ProtocolObject<dyn MTLBuffer>,- pub d: &'a ProtocolObject<dyn MTLBuffer>,- pub z: &'a ProtocolObject<dyn MTLBuffer>,- pub state: &'a ProtocolObject<dyn MTLBuffer>,- pub y: &'a ProtocolObject<dyn MTLBuffer>,- pub suffix_len: usize,- pub group_size: i32,- pub state_size: i32,- pub x_strides: [usize; 3],- pub dt_strides: [usize; 2],- pub cb_strides: [usize; 3],- pub state_strides: [usize; 3],- pub channels: usize,- pub head_dim: usize,-}--pub struct SSDPrefillKernels {- single: SSDPrefillMetalKernel,- single_64: SSDPrefill64MetalKernel,- sequential: SSDPrefillSequentialMetalKernel,-}--impl SSDPrefillKernels {- pub fn new(- context: &MetalContext,- data_type: DataType,- ) -> Result<Self, MetalError> {- let single = SSDPrefillMetalKernel::new(context, data_type)?;- let single_64 = SSDPrefill64MetalKernel::new(context, data_type)?;- let sequential = SSDPrefillSequentialMetalKernel::new(context, data_type)?;- Ok(Self {- single,- single_64,- sequential,- })- }-- pub fn encode(- &self,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- args: SSDPrefillArguments,- mode: SSDPrefillMode,- ) {- if mode == SSDPrefillMode::SinglePass {- if args.state_size == 64 {- self.single_64.encode(- &args.x.retain(),- &args.dt.retain(),- &args.b.retain(),- &args.c.retain(),- &args.d.retain(),- &args.z.retain(),- &args.state.retain(),- &args.y.retain(),- args.suffix_len as u32,- args.group_size,- args.state_size,- args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.channels as u32,- args.head_dim as u32,- compute_encoder,- )- } else {- self.single.encode(- &args.x.retain(),- &args.dt.retain(),- &args.b.retain(),- &args.c.retain(),- &args.d.retain(),- &args.z.retain(),- &args.state.retain(),- &args.y.retain(),- args.suffix_len as u32,- args.group_size,- args.state_size,- args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.channels as u32,- args.head_dim as u32,- compute_encoder,- )- }- } else if mode == SSDPrefillMode::Sequential {- self.sequential.encode(- &args.x.retain(),- &args.dt.retain(),- &args.b.retain(),- &args.c.retain(),- &args.d.retain(),- &args.z.retain(),- &args.state.retain(),- &args.y.retain(),- args.suffix_len as u32,- args.group_size,- args.state_size,- args.x_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.dt_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.cb_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.state_strides.iter().map(|x| *x as u32).collect::<Vec<_>>().as_slice(),- args.channels as u32,- args.head_dim as u32,- compute_encoder,- )- }- }-}diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rsindex 755f071eb..147f52c50 100644--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs@@ -140,7 +140,7 @@ pub struct LanguageModelGeneratorContext {pub model_config: LanguageModelConfig,pub decoder_config: Rc<DecoderConfig>,pub model_shape: ModelShape,- pub executables: Decoder,+ pub executables: Decoder<Metal>,pub kv_cache_update: Box<KVCacheUpdate<Metal>>,pub gpu_sampler: Sampling<Metal>,pub seed: PRng,diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rsindex 43a4e5dff..ac610f7bb 100644--- a/crates/uzu/src/classifier/mod.rs+++ b/crates/uzu/src/classifier/mod.rs@@ -7,7 +7,7 @@ pub use result::{ClassificationOutput, ClassificationStats};// Re-export context from metal backendpub use crate::backends::metal::ClassifierContext;// Re-export encodables from metal backend for convenience-pub use crate::backends::metal::encodable_block::{ClassifierLayer, ClassifierPredictionHead};-pub use crate::encodable_block::Pooling;+pub use crate::backends::metal::encodable_block::ClassifierLayer;+pub use crate::encodable_block::{ClassifierPredictionHead, Pooling};#[cfg(feature = "tracing")]pub use crate::forward_pass::traces::{ActivationTrace, LayerActivationTrace};diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rssimilarity index 86%rename from crates/uzu/src/backends/metal/encodable_block/attention.rsrename to crates/uzu/src/encodable_block/attention.rsindex bac0a64ec..dc3cc50b4 100644--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -2,24 +2,13 @@use std::collections::HashMap;-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};-use crate::{DataType,- backends::{- common::kernel::{+ backends::common::{+ Backend, CommandBuffer, Kernels,+ kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel, AttentionUpdateKVCacheKernel,- },- metal::{- Metal, MetalContext, MetalError,- kernel::{- attention::{AttentionGemmArguments, AttentionGemmBlock},- dsl::{- AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel,- AttentionUpdateKVCacheMetalKernel,- },- },+ attention::{AttentionGemmArguments, AttentionGemmBlock},},},encodable_block::{EncodableBlock, EncodingParameters},@@ -42,12 +31,12 @@ fn env_gemm_attention_enabled() -> bool {})}-pub struct Attention {- single_pass_kernels: HashMap<KernelKey, AttentionSinglePassMetalKernel>,- two_pass_1_kernels: HashMap<KernelKey, AttentionTwoPass1MetalKernel>,- two_pass_2_kernels: HashMap<u32, AttentionTwoPass2MetalKernel>,- update_kv_cache_kernel: AttentionUpdateKVCacheMetalKernel,- gemm_block: AttentionGemmBlock,+pub struct Attention<B: Backend> {+ single_pass_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionSinglePassKernel>,+ two_pass_1_kernels: HashMap<KernelKey, <B::Kernels as Kernels>::AttentionTwoPass1Kernel>,+ two_pass_2_kernels: HashMap<u32, <B::Kernels as Kernels>::AttentionTwoPass2Kernel>,+ update_kv_cache_kernel: <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel,+ gemm_block: AttentionGemmBlock<B>,layer_index: usize,attention_scale: Option<f32>,has_sinks: bool,@@ -55,16 +44,16 @@ pub struct Attention {sliding_window_size: Option<usize>,}-impl Attention {+impl<B: Backend> Attention<B> {pub fn new(- context: &MetalContext,+ context: &B::Context,data_type: DataType,layer_index: usize,attention_scale: Option<f32>,has_sinks: bool,is_causal: bool,sliding_window_size: Option<usize>,- ) -> Result<Self, MetalError> {+ ) -> Result<Self, B::Error> {let mut single_pass_kernels = HashMap::new();let mut two_pass_1_kernels = HashMap::new();let mut two_pass_2_kernels = HashMap::new();@@ -78,22 +67,22 @@ impl Attention {};let float_mask = has_mask;- let sp_kernel = AttentionSinglePassMetalKernel::new(+ let sp_kernel = <B::Kernels as Kernels>::AttentionSinglePassKernel::new(context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,)?;single_pass_kernels.insert(key, sp_kernel);- let tp1_kernel = AttentionTwoPass1MetalKernel::new(+ let tp1_kernel = <B::Kernels as Kernels>::AttentionTwoPass1Kernel::new(context, data_type, head_dim, float_mask, has_mask, has_sinks, is_causal,)?;two_pass_1_kernels.insert(key, tp1_kernel);- let tp2_kernel = AttentionTwoPass2MetalKernel::new(context, data_type, head_dim)?;+ let tp2_kernel = <B::Kernels as Kernels>::AttentionTwoPass2Kernel::new(context, data_type, head_dim)?;two_pass_2_kernels.insert(head_dim, tp2_kernel);}}- let update_kv_cache_kernel = AttentionUpdateKVCacheMetalKernel::new(context, data_type)?;+ let update_kv_cache_kernel = <B::Kernels as Kernels>::AttentionUpdateKVCacheKernel::new(context, data_type)?;let gemm_block = AttentionGemmBlock::new(data_type);Ok(Self {@@ -138,20 +127,17 @@ impl Attention {}}-impl EncodableBlock<Metal> for Attention {+impl<B: Backend> EncodableBlock<B> for Attention<B> {fn encode(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {- let compute_encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &compute_encoder);- compute_encoder.end_encoding();+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, &encoder));if parameters.wait_until_completed {- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();}}@@ -162,9 +148,9 @@ impl EncodableBlock<Metal> for Attention {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- compute_encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ compute_encoder: &B::ComputeEncoder,) {let (suffix_length, num_heads, head_dim, num_groups, max_sequence_length) = {let qkv_binding = state.arrays(&[ArrayId::QKV]);@@ -356,12 +342,12 @@ impl EncodableBlock<Metal> for Attention {match variant {KernelVariant::Gemm => {let args = AttentionGemmArguments {- queries_buffer: &queries_buffer,+ queries_buffer,keys_buffer: &key_cache_buffer,values_buffer: &value_cache_buffer,- output_buffer: &attention_output_buffer,- mask_buffer: attention_bias_buffer.as_deref(),- sinks_buffer: sinks_buffer.as_deref(),+ output_buffer: attention_output_buffer,+ mask_buffer: attention_bias_buffer.as_ref(),+ sinks_buffer: sinks_buffer.as_ref(),num_heads,num_groups,suffix_length,@@ -373,13 +359,13 @@ impl EncodableBlock<Metal> for Attention {scale,};self.gemm_block- .encode(state.mtl_context(), compute_encoder, &args)+ .encode(state.context(), compute_encoder, &args).expect("Failed to encode AttentionGemmBlock");},KernelVariant::SinglePass => {let kernel = match self.single_pass_kernels.get(&kernel_key) {Some(k) => k,- None => panic!("Can not find AttentionSinglePassMetalKernel for key {:?}", kernel_key),+ None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),};let mask_buffer_opt = attention_bias_buffer.as_ref();let sinks_buffer_opt = sinks_buffer.as_ref();@@ -408,11 +394,11 @@ impl EncodableBlock<Metal> for Attention {KernelVariant::TwoPass => {let kernel_pass1 = match self.two_pass_1_kernels.get(&kernel_key) {Some(k) => k,- None => panic!("Can not find AttentionTwoPass1MetalKernel for key {:?}", kernel_key),+ None => panic!("Can not find AttentionTwoPass1Kernel for key {:?}", kernel_key),};let kernel_pass2 = match self.two_pass_2_kernels.get(&(head_dim as u32)) {Some(k) => k,- None => panic!("Can not find AttentionTwoPass2MetalKernel for key {:?}", kernel_key),+ None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),};let mask_buffer_opt = attention_bias_buffer.as_ref().map(|b| b);diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rsindex 6e10e9855..5a9444f27 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs@@ -122,7 +122,7 @@ wherelet element_size = input_array.data_type().size_in_bytes();self.kernel.borrow_mut().encode(- state.mtl_context(),+ state.context(),encoder,FullPrecisionMatmulArguments {a: input_array.buffer(),diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rsindex 17b8e83a6..56d32d294 100644--- a/crates/uzu/src/encodable_block/full_precision_linear.rs+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs@@ -148,7 +148,7 @@ wherelet output_array = arrays[1].borrow_mut();self.kernel.borrow_mut().encode(- state.mtl_context(),+ state.context(),encoder,FullPrecisionMatmulArguments {a: input_array.buffer(),diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rssimilarity index 75%rename from crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsrename to crates/uzu/src/encodable_block/mamba_mixer.rsindex 182533c19..bcaa39755 100644--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -2,54 +2,47 @@use std::env;-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};-use objc2::{Message, rc::Retained, runtime::ProtocolObject};--use super::transformer_layer;use crate::{Activation, DataType,array::Array,backends::{- common::kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel},- metal::{- Metal, MetalContext,+ common::{+ Backend, CommandBuffer, Kernels,kernel::{- dsl::{- Conv1dDecodeMetalKernel, Conv1dPackMetalKernel, Conv1dScanMetalKernel, SSDUpdateMetalKernel,- SplitInProjMetalKernel,- },- ssm::ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},+ Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},+ metal::{Metal, encodable_block::transformer_layer},},config::{DecoderLayerType, Mamba2Config},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,+ parameters::{ParameterTree, resolve_subtree},};-pub(crate) struct MambaMixer {+pub(crate) struct MambaMixer<B: Backend> {layer_index: usize,config: Mamba2Config,- in_projection: Box<dyn EncodableBlock<Metal>>,- out_projection: Box<dyn EncodableBlock<Metal>>,- split_inproj: SplitInProjMetalKernel,- conv_decode: Conv1dDecodeMetalKernel,- conv_pack: Conv1dPackMetalKernel,- conv_scan: Conv1dScanMetalKernel,- ssd_prefill: SSDPrefillKernels,- ssd_update: SSDUpdateMetalKernel,- conv_weight: Array<Metal>,- conv_bias: Option<Array<Metal>>,- gate_bias: Array<Metal>,- skip_connection_weight: Array<Metal>,+ in_projection: Box<dyn EncodableBlock<B>>,+ out_projection: Box<dyn EncodableBlock<B>>,+ split_inproj: <B::Kernels as Kernels>::SplitInProjKernel,+ conv_decode: <B::Kernels as Kernels>::Conv1dDecodeKernel,+ conv_pack: <B::Kernels as Kernels>::Conv1dPackKernel,+ conv_scan: <B::Kernels as Kernels>::Conv1dScanKernel,+ ssd_prefill: SSDPrefillKernels<B>,+ ssd_update: <B::Kernels as Kernels>::SSDUpdateKernel,+ conv_weight: Array<B>,+ conv_bias: Option<Array<B>>,+ gate_bias: Array<B>,+ skip_connection_weight: Array<B>,prefill_mode: SSDPrefillMode,}-impl MambaMixer {+impl MambaMixer<Metal> {#[allow(clippy::too_many_arguments)]pub(crate) fn new(- mtl_context: &MetalContext,+ context: &<Metal as Backend>::Context,layer_type: DecoderLayerType,mamba_config: Mamba2Config,layer_index: usize,@@ -57,7 +50,7 @@ impl MambaMixer {num_heads: usize,head_dim: usize,num_groups: usize,- decoder_layer_loader: &ParameterTree<MetalContext>,+ decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,) -> Self {let _ = (num_heads, head_dim, num_groups);if !matches!(layer_type, DecoderLayerType::StateSpace { .. }) {@@ -73,7 +66,7 @@ impl MambaMixer {mamba_config.has_in_biases,model_dim,[mamba_config.conv_dim(), mamba_config.inner_dim(), mamba_config.num_heads],- mtl_context,+ context,&resolve_subtree(decoder_layer_loader, &["mixer.in_projection", "mixer.in_proj"]),ArrayId::Main,ArrayId::SsmInProj,@@ -85,7 +78,7 @@ impl MambaMixer {mamba_config.has_out_biases,mamba_config.inner_dim(),[model_dim],- mtl_context,+ context,&resolve_subtree(decoder_layer_loader, &["mixer.out_projection", "mixer.out_proj"]),ArrayId::AttentionOutput,ArrayId::Main,@@ -101,25 +94,27 @@ impl MambaMixer {let gate_bias = split_tree.leaf("gate_bias").unwrap().clone();let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap().clone();- let split_inproj =- SplitInProjMetalKernel::new(mtl_context, data_type).expect("Failed to create split in-projection kernel");- let conv_scan = Conv1dScanMetalKernel::new(- mtl_context,+ let split_inproj = <<Metal as Backend>::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)+ .expect("Failed to create split in-projection kernel");+ let conv_scan = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(+ context,data_type,Self::activation_to_int(&mamba_config.activation),mamba_config.conv_config.has_biases,).expect("Failed to create conv scan kernel");- let conv_decode = Conv1dDecodeMetalKernel::new(- mtl_context,+ let conv_decode = <<Metal as Backend>::Kernels as Kernels>::Conv1dDecodeKernel::new(+ context,data_type,Self::activation_to_int(&mamba_config.activation),mamba_config.conv_config.has_biases,).expect("Failed to create conv decode kernel");- let conv_pack = Conv1dPackMetalKernel::new(mtl_context, data_type).expect("Failed to create conv pack kernel");- let ssd_prefill = SSDPrefillKernels::new(mtl_context, data_type).expect("Failed to create SSD prefill kernel");- let ssd_update = SSDUpdateMetalKernel::new(mtl_context, data_type).expect("Failed to create SSD decode kernel");+ let conv_pack = <<Metal as Backend>::Kernels as Kernels>::Conv1dPackKernel::new(context, data_type)+ .expect("Failed to create conv pack kernel");+ let ssd_prefill = SSDPrefillKernels::new(context, data_type).expect("Failed to create SSD prefill kernel");+ let ssd_update = <<Metal as Backend>::Kernels as Kernels>::SSDUpdateKernel::new(context, data_type)+ .expect("Failed to create SSD decode kernel");let prefill_mode = resolve_prefill_mode_from_env();Self {@@ -140,12 +135,14 @@ impl MambaMixer {prefill_mode,}}+}+impl<B: Backend> MambaMixer<B> {fn encode_pipeline_with_encoder(&self,- state: &mut ForwardPassState<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters<Metal>,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,+ parameters: &EncodingParameters<B>,) {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {@@ -167,8 +164,8 @@ impl MambaMixer {fn run_split_inproj(&self,- state: &mut ForwardPassState<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,suffix_length: usize,) {let arrays = state.arrays(&[@@ -182,12 +179,12 @@ impl MambaMixer {let gate = arrays[2].borrow_mut();let dt = arrays[3].borrow_mut();- let input_buf = in_proj.buffer().to_owned();- let conv_buf = conv_inputs.buffer().to_owned();- let gate_buf = gate.buffer().to_owned();- let dt_buf = dt.buffer().to_owned();+ let input_buf = in_proj.buffer().clone();+ let conv_buf = conv_inputs.buffer().clone();+ let gate_buf = gate.buffer().clone();+ let dt_buf = dt.buffer().clone();let gate_bias = self.gate_bias.clone();- let bias_buf = gate_bias.buffer().to_owned();+ let bias_buf = gate_bias.buffer().clone();let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();@@ -211,8 +208,8 @@ impl MambaMixer {fn run_conv_scan(&self,- state: &mut ForwardPassState<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,suffix_length: usize,) {let arrays = state.arrays(&[@@ -228,11 +225,11 @@ impl MambaMixer {let b_arr = arrays[3].borrow_mut();let c_arr = arrays[4].borrow_mut();- let input_buf = conv_inputs.buffer().to_owned();+ let input_buf = conv_inputs.buffer().clone();let state_buf = conv_state.buffer().clone();- let x_buf = x_arr.buffer().to_owned();- let b_buf = b_arr.buffer().to_owned();- let c_buf = c_arr.buffer().to_owned();+ let x_buf = x_arr.buffer().clone();+ let b_buf = b_arr.buffer().clone();+ let c_buf = c_arr.buffer().clone();let weight_storage = self.conv_weight.clone();let weight_buf = weight_storage.buffer().clone();@@ -289,7 +286,7 @@ impl MambaMixer {};if conv_dim > 0 && self.config.kernel_size > 0 {- let conv_source = padded_buf.as_deref().unwrap_or(&input_buf).retain();+ let conv_source = padded_buf.as_ref().unwrap_or(&input_buf).clone();self.conv_scan.encode(&conv_source,&weight_buf,@@ -313,8 +310,8 @@ impl MambaMixer {fn run_prefill_ssm(&self,- state: &mut ForwardPassState<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,suffix_length: usize,) {let base_arrays = state.arrays(&[@@ -378,8 +375,8 @@ impl MambaMixer {fn run_decode_ssm(&self,- state: &mut ForwardPassState<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,suffix_length: usize,) {let arrays = state.arrays(&[@@ -471,21 +468,19 @@ fn resolve_prefill_mode_from_env() -> SSDPrefillMode {}}-impl EncodableBlock<Metal> for MambaMixer {+impl<B: Backend> EncodableBlock<B> for MambaMixer<B> {fn encode(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {if self.supports_shared_encoder() {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_pipeline_with_encoder(state, &encoder, parameters);- encoder.end_encoding();+ command_buffer+ .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, encoder, parameters));if parameters.wait_until_completed {- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();}return;@@ -498,20 +493,20 @@ impl EncodableBlock<Metal> for MambaMixer {self.in_projection.encode(state, parameters, command_buffer);- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.run_split_inproj(state, &encoder, active_suffix_length);- self.run_conv_scan(state, &encoder, active_suffix_length);- if active_suffix_length == 1 {- self.run_decode_ssm(state, &encoder, active_suffix_length);- } else {- self.run_prefill_ssm(state, &encoder, active_suffix_length);- }- encoder.end_encoding();+ command_buffer.with_compute_encoder(|encoder| {+ self.run_split_inproj(state, encoder, active_suffix_length);+ self.run_conv_scan(state, encoder, active_suffix_length);+ if active_suffix_length == 1 {+ self.run_decode_ssm(state, encoder, active_suffix_length);+ } else {+ self.run_prefill_ssm(state, encoder, active_suffix_length);+ }+ });self.out_projection.encode(state, parameters, command_buffer);if parameters.wait_until_completed {- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();}}@@ -522,24 +517,10 @@ impl EncodableBlock<Metal> for MambaMixer {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,) {self.encode_pipeline_with_encoder(state, encoder, parameters);}}--fn resolve_subtree<'tree>(- loader: &'tree ParameterTree<MetalContext>,- candidates: &[&str],-) -> ParameterTree<'tree, MetalContext> {- for candidate in candidates {- if let Ok(tree) = loader.subtree(candidate) {- return tree;- }- }-- let missing = candidates.first().copied().unwrap_or("<missing subtree name>");- loader.subtree(missing).unwrap_or_else(|_| panic!("Unable to resolve parameter subtree '{missing}'"))-}diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/encodable_block/mlp.rssimilarity index 59%rename from crates/uzu/src/backends/metal/encodable_block/mlp.rsrename to crates/uzu/src/encodable_block/mlp.rsindex 6cfcd4bc0..9a2af53a4 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/encodable_block/mlp.rs@@ -1,25 +1,22 @@//! MLP block encodable.-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};-use crate::{- backends::metal::{Metal, kernel::mlp::MlpGateActMulEncodable},+ backends::common::{Backend, CommandBuffer, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},};-pub struct MlpBlock {- up: Box<dyn EncodableBlock<Metal>>,- gate: MlpGateActMulEncodable,- down: Box<dyn EncodableBlock<Metal>>,+pub struct MlpBlock<B: Backend> {+ up: Box<dyn EncodableBlock<B>>,+ gate: MlpGateActMulEncodable<B>,+ down: Box<dyn EncodableBlock<B>>,}-impl MlpBlock {+impl<B: Backend> MlpBlock<B> {pub fn new(- up: Box<dyn EncodableBlock<Metal>>,- gate: MlpGateActMulEncodable,- down: Box<dyn EncodableBlock<Metal>>,+ up: Box<dyn EncodableBlock<B>>,+ gate: MlpGateActMulEncodable<B>,+ down: Box<dyn EncodableBlock<B>>,) -> Self {Self {up,@@ -29,24 +26,21 @@ impl MlpBlock {}}-impl EncodableBlock<Metal> for MlpBlock {+impl<B: Backend> EncodableBlock<B> for MlpBlock<B> {fn encode(&self,- state: &mut ForwardPassState<Metal>,- params: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ params: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {if self.supports_shared_encoder() {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, params, &encoder);- encoder.end_encoding();+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, params, encoder));} else {// Upself.up.encode(state, params, command_buffer);// Gate act+mul (fused_up -> hidden)- {+ command_buffer.with_compute_encoder(|encoder| {let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);let fused = arrays[0].borrow_mut();let hidden = arrays[1].borrow_mut();@@ -54,20 +48,17 @@ impl EncodableBlock<Metal> for MlpBlock {let fused_buf = fused.buffer();let hidden_buf = hidden.buffer();- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");self.gate- .encode(&encoder, fused_buf, hidden_buf, m)+ .encode(encoder, fused_buf, hidden_buf, m).expect("Failed to encode MLP activation/mul kernel");- encoder.end_encoding();- }+ });// Downself.down.encode(state, params, command_buffer);}if params.wait_until_completed {- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();}}@@ -78,9 +69,9 @@ impl EncodableBlock<Metal> for MlpBlock {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState<Metal>,- params: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ params: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,) {// Upself.up.encode_with_shared_encoder(state, params, encoder);diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 79b75fec2..050224bc2 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -4,34 +4,44 @@ use crate::{};mod activation;+mod attention;mod full_precision_embedding_lookup;mod full_precision_embedding_readout;mod full_precision_linear;mod layer_norm;+mod mamba_mixer;+mod mlp;mod normalization;mod pooling;+mod prediction_head;mod qk_norm;mod quantized_embedding_lookup;mod quantized_linear;mod rms_norm;mod rope;mod sampling;+mod short_conv_mixer;mod tensor_add_swap;mod tensor_copy;pub use activation::Activation;+pub use attention::Attention;pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};pub use layer_norm::{LayerNorm, LayerNormError};+pub(crate) use mamba_mixer::MambaMixer;+pub use mlp::MlpBlock;pub use normalization::{Normalization, NormalizationError};pub use pooling::Pooling;+pub use prediction_head::ClassifierPredictionHead;pub use qk_norm::{QKNorm, QKNormError};pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};pub use rms_norm::{RMSNorm, RMSNormError};pub use rope::Rope;pub use sampling::Sampling;+pub use short_conv_mixer::ShortConvMixer;pub use tensor_add_swap::TensorAddSwap;pub use tensor_copy::TensorCopy;diff --git a/crates/uzu/src/encodable_block/prediction_head.rs b/crates/uzu/src/encodable_block/prediction_head.rsnew file mode 100644index 000000000..fb3b12d61--- /dev/null+++ b/crates/uzu/src/encodable_block/prediction_head.rs@@ -0,0 +1,88 @@+//! Prediction head encodable for classification output.++#[cfg(feature = "tracing")]+use crate::forward_pass::state::ArrayId;+use crate::{+ backends::common::{Backend, CommandBuffer},+ encodable_block::{EncodableBlock, EncodingParameters},+ forward_pass::state::ForwardPassState,+};++pub struct ClassifierPredictionHead<B: Backend> {+ dense: Box<dyn EncodableBlock<B>>,+ activation: Box<dyn EncodableBlock<B>>,+ norm: Box<dyn EncodableBlock<B>>,+ readout: Box<dyn EncodableBlock<B>>,+ #[allow(dead_code)]+ num_labels: usize,+}++impl<B: Backend> ClassifierPredictionHead<B> {+ pub fn new(+ dense: Box<dyn EncodableBlock<B>>,+ activation: Box<dyn EncodableBlock<B>>,+ norm: Box<dyn EncodableBlock<B>>,+ readout: Box<dyn EncodableBlock<B>>,+ num_labels: usize,+ ) -> Self {+ Self {+ dense,+ activation,+ norm,+ readout,+ num_labels,+ }+ }+}++impl<B: Backend> EncodableBlock<B> for ClassifierPredictionHead<B> {+ fn encode(+ &self,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,+ ) {+ if self.supports_shared_encoder() {+ command_buffer.with_compute_encoder(|encoder| self.encode_with_shared_encoder(state, parameters, encoder));+ } else {+ self.dense.encode(state, parameters, command_buffer);+ self.activation.encode(state, parameters, command_buffer);+ self.norm.encode(state, parameters, command_buffer);+ self.readout.encode(state, parameters, command_buffer);+ }++ #[cfg(feature = "tracing")]+ {+ let traces = state.traces().clone();+ state.encode_copy_array(+ command_buffer,+ ArrayId::ClassifierPredictionHeadLogits,+ traces.borrow().logits.clone(),+ );+ }++ if parameters.wait_until_completed {+ command_buffer.submit();+ command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ self.dense.supports_shared_encoder()+ && self.activation.supports_shared_encoder()+ && self.norm.supports_shared_encoder()+ && self.readout.supports_shared_encoder()+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,+ ) {+ self.dense.encode_with_shared_encoder(state, parameters, encoder);+ self.activation.encode_with_shared_encoder(state, parameters, encoder);+ self.norm.encode_with_shared_encoder(state, parameters, encoder);+ self.readout.encode_with_shared_encoder(state, parameters, encoder);+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rssimilarity index 74%rename from crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsrename to crates/uzu/src/encodable_block/short_conv_mixer.rsindex 7293db9d7..4582af4e7 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -1,64 +1,41 @@-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::transformer_layer;use crate::{DataType,array::Array,backends::{common::{- Context,+ Backend, CommandBuffer, Context, Kernels,kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},},- metal::{- Metal, MetalContext,- kernel::dsl::{- ShortConvDecodeMetalKernel, ShortConvPackMetalKernel, ShortConvPrefillMetalKernel,- ShortConvTrieMetalKernel,- },- },+ metal::{Metal, encodable_block::transformer_layer},},config::{DecoderLayerType, ShortConvConfig},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,+ parameters::{ParameterTree, resolve_subtree},};-pub(crate) struct ShortConvMixer {+pub struct ShortConvMixer<B: Backend> {layer_index: usize,config: ShortConvConfig,model_dim: usize,- in_projection: Box<dyn EncodableBlock<Metal>>,- out_projection: Box<dyn EncodableBlock<Metal>>,- short_conv_pack: ShortConvPackMetalKernel,- short_conv_prefill: ShortConvPrefillMetalKernel,- short_conv_decode: ShortConvDecodeMetalKernel,- short_conv_trie: ShortConvTrieMetalKernel,- conv_weight: Array<Metal>,- conv_bias: Option<Array<Metal>>,+ in_projection: Box<dyn EncodableBlock<B>>,+ out_projection: Box<dyn EncodableBlock<B>>,+ short_conv_pack: <B::Kernels as Kernels>::ShortConvPackKernel,+ short_conv_prefill: <B::Kernels as Kernels>::ShortConvPrefillKernel,+ short_conv_decode: <B::Kernels as Kernels>::ShortConvDecodeKernel,+ short_conv_trie: <B::Kernels as Kernels>::ShortConvTrieKernel,+ conv_weight: Array<B>,+ conv_bias: Option<Array<B>>,}-fn resolve_subtree<'tree>(- tree: &'tree ParameterTree<MetalContext>,- candidates: &[&str],-) -> ParameterTree<'tree, MetalContext> {- for candidate in candidates {- if let Ok(subtree) = tree.subtree(candidate) {- return subtree;- }- }- panic!("Could not find any of {:?} in parameter tree", candidates);-}--impl ShortConvMixer {- #[allow(clippy::too_many_arguments)]- pub(crate) fn new(- mtl_context: &MetalContext,+impl ShortConvMixer<Metal> {+ pub fn new(+ context: &<Metal as Backend>::Context,layer_type: DecoderLayerType,short_conv_config: ShortConvConfig,layer_index: usize,model_dim: usize,- decoder_layer_loader: &ParameterTree<MetalContext>,+ decoder_layer_loader: &ParameterTree<<Metal as Backend>::Context>,) -> Self {if !matches!(layer_type, DecoderLayerType::ShortConv { .. }) {panic!("Layer {} marked as non-ShortConv but ShortConv config provided", layer_index);@@ -74,7 +51,7 @@ impl ShortConvMixer {false,model_dim,[model_dim * 3],- mtl_context,+ context,&resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),ArrayId::Main,ArrayId::SsmInProj,@@ -86,7 +63,7 @@ impl ShortConvMixer {false,model_dim,[model_dim],- mtl_context,+ context,&resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),ArrayId::AttentionOutput,ArrayId::Main,@@ -101,14 +78,17 @@ impl ShortConvMixer {};let has_bias = short_conv_config.conv_config.has_biases;- let short_conv_pack =- ShortConvPackMetalKernel::new(mtl_context, data_type).expect("Failed to create short conv pack kernel");- let short_conv_prefill = ShortConvPrefillMetalKernel::new(mtl_context, data_type, has_bias)- .expect("Failed to create short conv prefill kernel");- let short_conv_decode = ShortConvDecodeMetalKernel::new(mtl_context, data_type, has_bias)- .expect("Failed to create short conv decode kernel");- let short_conv_trie = ShortConvTrieMetalKernel::new(mtl_context, data_type, has_bias)- .expect("Failed to create short conv trie kernel");+ let short_conv_pack = <<Metal as Backend>::Kernels as Kernels>::ShortConvPackKernel::new(context, data_type)+ .expect("Failed to create short conv pack kernel");+ let short_conv_prefill =+ <<Metal as Backend>::Kernels as Kernels>::ShortConvPrefillKernel::new(context, data_type, has_bias)+ .expect("Failed to create short conv prefill kernel");+ let short_conv_decode =+ <<Metal as Backend>::Kernels as Kernels>::ShortConvDecodeKernel::new(context, data_type, has_bias)+ .expect("Failed to create short conv decode kernel");+ let short_conv_trie =+ <<Metal as Backend>::Kernels as Kernels>::ShortConvTrieKernel::new(context, data_type, has_bias)+ .expect("Failed to create short conv trie kernel");Self {layer_index,@@ -124,12 +104,14 @@ impl ShortConvMixer {conv_bias,}}+}+impl<B: Backend> ShortConvMixer<B> {fn encode_pipeline(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {@@ -138,23 +120,21 @@ impl ShortConvMixer {self.in_projection.encode(state, parameters, command_buffer);- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.run_conv(state, &encoder, active_suffix_length);- encoder.end_encoding();+ command_buffer.with_compute_encoder(|encoder| self.run_conv(state, &encoder, active_suffix_length));self.out_projection.encode(state, parameters, command_buffer);if parameters.wait_until_completed {- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();}}fn encode_pipeline_with_encoder(&self,- state: &mut ForwardPassState<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- parameters: &EncodingParameters<Metal>,+ state: &mut ForwardPassState<B>,+ encoder: &B::ComputeEncoder,+ parameters: &EncodingParameters<B>,) {let active_suffix_length = state.active_suffix_length();if active_suffix_length == 0 {@@ -170,7 +150,7 @@ impl ShortConvMixer {fn clear_suffix_state_valid_range(&self,- state: &ForwardPassState<Metal>,+ state: &ForwardPassState<B>,) {let Some(cache_layers) = state.cache_layers() else {return;@@ -182,7 +162,7 @@ impl ShortConvMixer {fn set_suffix_state_valid_range(&self,- state: &ForwardPassState<Metal>,+ state: &ForwardPassState<B>,start: usize,len: usize,) {@@ -196,8 +176,8 @@ impl ShortConvMixer {fn run_conv(&self,- state: &mut ForwardPassState<Metal>,- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ compute: &B::ComputeEncoder,active_suffix_length: usize,) {self.clear_suffix_state_valid_range(state);@@ -235,8 +215,8 @@ impl ShortConvMixer {fn run_prefill_conv(&self,- state: &mut ForwardPassState<Metal>,- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ compute: &B::ComputeEncoder,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -265,7 +245,7 @@ impl ShortConvMixer {let element_size = data_type.size_in_bytes();let padded_rows = state_stride + suffix_length;let padded_size = padded_rows * self.model_dim * element_size;- let padded_buf = state.mtl_context().create_buffer(padded_size).expect("Failed to create padded buffer");+ let padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");self.short_conv_pack.encode(&state_buf,&in_proj_buf,@@ -295,8 +275,8 @@ impl ShortConvMixer {fn run_trie_conv(&self,- state: &mut ForwardPassState<Metal>,- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ compute: &B::ComputeEncoder,sampling_start: usize,trie_len: usize,) {@@ -360,8 +340,8 @@ impl ShortConvMixer {fn run_decode_conv(&self,- state: &mut ForwardPassState<Metal>,- compute: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ compute: &B::ComputeEncoder,suffix_length: usize,) {if self.model_dim == 0 || suffix_length == 0 {@@ -402,21 +382,19 @@ impl ShortConvMixer {}}-impl EncodableBlock<Metal> for ShortConvMixer {+impl<B: Backend> EncodableBlock<B> for ShortConvMixer<B> {fn encode(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ command_buffer: &B::CommandBuffer,) {if self.supports_shared_encoder() {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_pipeline_with_encoder(state, &encoder, parameters);- encoder.end_encoding();+ command_buffer+ .with_compute_encoder(|encoder| self.encode_pipeline_with_encoder(state, &encoder, parameters));if parameters.wait_until_completed {- command_buffer.commit();+ command_buffer.submit();command_buffer.wait_until_completed();}} else {@@ -430,9 +408,9 @@ impl EncodableBlock<Metal> for ShortConvMixer {fn encode_with_shared_encoder(&self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,) {self.encode_pipeline_with_encoder(state, encoder, parameters);}diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex 02c1a076f..7967ea70f 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -448,7 +448,7 @@ impl<B: Backend> ForwardPassState<B> {// Accessors// ========================================================================- pub fn mtl_context(&self) -> &B::Context {+ pub fn context(&self) -> &B::Context {self.context.as_ref()}diff --git a/crates/uzu/src/parameters/loader.rs b/crates/uzu/src/parameters/loader.rsindex ae0b098ed..be1380602 100644--- a/crates/uzu/src/parameters/loader.rs+++ b/crates/uzu/src/parameters/loader.rs@@ -193,3 +193,15 @@ impl<'loader, C: Context> ParameterTree<'loader, C> {self.loader.read_extract_at(&self.join_prefix(name), buf, shape, data_type)}}++pub fn resolve_subtree<'tree, C: Context>(+ tree: &'tree ParameterTree<C>,+ candidates: &[&str],+) -> ParameterTree<'tree, C> {+ for candidate in candidates {+ if let Ok(subtree) = tree.subtree(candidate) {+ return subtree;+ }+ }+ panic!("Could not find any of {:?} in parameter tree", candidates);+}diff --git a/crates/uzu/src/parameters/mod.rs b/crates/uzu/src/parameters/mod.rsindex ff3f5ddc2..903847308 100644--- a/crates/uzu/src/parameters/mod.rs+++ b/crates/uzu/src/parameters/mod.rs@@ -1,5 +1,5 @@mod loader;-pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree};+pub use loader::{ParameterLoader, ParameterLoaderError, ParameterTree, resolve_subtree};mod safetensors_metadata;// Re-export the safetensors header reader so other modules (e.g. decoderdiff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rsindex 3cec1dbd3..40838afea 100644--- a/crates/uzu/tests/attention_kernel_test.rs+++ b/crates/uzu/tests/attention_kernel_test.rs@@ -13,14 +13,14 @@ use uzu::{backends::{common::{Context,- kernel::{AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel},- },- metal::{- MetalContext,kernel::{+ AttentionSinglePassKernel, AttentionTwoPass1Kernel, AttentionTwoPass2Kernel,attention::{AttentionGemmArguments, AttentionGemmBlock},- dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},},+ },+ metal::{+ Metal, MetalContext,+ kernel::dsl::{AttentionSinglePassMetalKernel, AttentionTwoPass1MetalKernel, AttentionTwoPass2MetalKernel},metal_extensions::CommandBufferTimingExt,},},@@ -439,7 +439,7 @@ fn run_single_pass_attention_with_is_causal(}fn run_gemm_attention(- kernel: &AttentionGemmBlock,+ kernel: &AttentionGemmBlock<Metal>,context: &MetalContext,queries: &Array4<f32>,keys: &Array4<f32>,@@ -472,8 +472,8 @@ fn run_gemm_attention(keys_buffer: &key_cache_buffer,values_buffer: &value_cache_buffer,output_buffer: &output_buffer,- mask_buffer: mask_buffer.as_deref(),- sinks_buffer: sinks_buffer.as_deref(),+ mask_buffer: mask_buffer.as_ref(),+ sinks_buffer: sinks_buffer.as_ref(),num_heads,num_groups: num_kv_heads,suffix_length: seq_len,diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex ddaa4ff4d..436d552e7 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -11,14 +11,14 @@ use objc2::runtime::ProtocolObject;use uzu::{DataType,backends::{- common::{Context, kernel::Conv1dScanKernel},- metal::{- MetalContext,+ common::{+ Context,kernel::{- dsl::Conv1dScanMetalKernel,- ssm::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},+ Conv1dScanKernel,+ ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},+ metal::{Metal, MetalContext, kernel::dsl::Conv1dScanMetalKernel},},};@@ -199,7 +199,7 @@ impl SSDPrefillFixture {fn run_prefill_kernel_mode(ctx: &MetalContext,- kernel: &SSDPrefillKernels,+ kernel: &SSDPrefillKernels<Metal>,fixture: &SSDPrefillFixture,mode: SSDPrefillMode,) -> (Vec<f32>, Vec<f32>, Option<(Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>)>) {@@ -228,7 +228,7 @@ fn run_prefill_kernel_mode(write_buffer(&state_buf, &fixture.state_init);zero_buffer(&y_buf);- let args = SSDPrefillArguments {+ let args = SSDPrefillArguments::<Metal> {x: &x_buf,dt: &dt_buf,b: &b_buf,@@ -363,7 +363,7 @@ fn assert_deterministic_for_mode(mode: SSDPrefillMode) {eprintln!("Skipping SSD prefill determinism test: no Metal device");return;};- let kernel = SSDPrefillKernels::new(&ctx, DataType::F32).unwrap();+ let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();let fixture = SSDPrefillFixture::new();let (y_a, state_a, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);@@ -378,7 +378,7 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {eprintln!("Skipping SSD prefill reference test: no Metal device");return;};- let kernel = SSDPrefillKernels::new(&ctx, DataType::F32).unwrap();+ let kernel = SSDPrefillKernels::<Metal>::new(&ctx, DataType::F32).unwrap();let fixture = SSDPrefillFixture::new();let (y_ref, state_ref) = ssd_prefill_cpu_reference(