A high-performance inference engine for AI models
remove mpsgraph
eugene8aa9ea685e36
diff --git a/Cargo.lock b/Cargo.lockindex c3df3983a..b5369e596 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2191,19 +2191,6 @@ dependencies = ["cxx",]-[[package]]-name = "mpsgraph"-version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b3d4f4ee48894ff2d83808e8a0333f00eb7576375a5a77e8132f10ebb2a1f86"-dependencies = [- "bitflags 2.9.4",- "block2",- "metal",- "objc2",- "objc2-foundation",-]-[[package]]name = "multer"version = "3.1.0"@@ -4103,7 +4090,6 @@ dependencies = ["metal","minijinja","minijinja-contrib",- "mpsgraph","ndarray","num-traits","objc2",diff --git a/crates/uzu/src/backends/metal/array.rs b/crates/uzu/src/backends/metal/array.rsindex 5ecbcea0d..555a70906 100644--- a/crates/uzu/src/backends/metal/array.rs+++ b/crates/uzu/src/backends/metal/array.rs@@ -2,8 +2,6 @@ use std::ops::Range;use half::{bf16, f16};use metal::{Buffer as MTLBuffer, MTLResourceOptions};-use mpsgraph::TensorData;-use objc2::rc::Retained;use crate::{Array, ArrayElement, DataType, array::array_size_in_bytes};@@ -80,15 +78,6 @@ impl MetalArray {self.offset}- pub unsafe fn to_mps_tensor_data(&mut self) -> Retained<TensorData> {- TensorData::new_with_mtl_buffer(- &self.buffer,- &self.shape,- self.data_type.into(),- None,- )- }-pub unsafe fn mtl_buffer(&mut self) -> &MTLBuffer {&self.buffer}diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex c48875e70..567507df0 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -1,24 +1,18 @@-use std::{- cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,- rc::Rc,-};+use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};-use mpsgraph::{CommandBuffer as MPSCommandBuffer, Device as MPSDevice, Graph};-use objc2::rc::{Retained, autoreleasepool};+use metal::CommandBuffer;use super::{KernelDataType, MTLContext, ModelShape,compilation_parameters::CompilationConfig,encodable_block::{- ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling,- Rope,+ Activation, ClassifierLayer, ClassifierPredictionHead, Normalization,+ Pooling, Rope,transformer_layer::{embed_block, linear_block},},forward_pass::{- ArrayId, EncodableBlock, IOArrays, MPSGraphBlock, RopeType,- ScratchBuffers, SharedBuffers,+ ArrayId, EncodableBlock, RopeType, ScratchBuffers, SharedBuffers,},- graph::{common::activation, placeholder, shaped_type},kernel::{PoolingKernel, SigmoidKernel},};use crate::{@@ -31,7 +25,7 @@ use crate::{pub struct ClassifierContext {pub mtl_context: Rc<MTLContext>,- pub command_buffer: Retained<MPSCommandBuffer>,+ pub command_buffer: CommandBuffer,pub shared_buffers: Rc<RefCell<SharedBuffers>>,pub scratch_buffers: ScratchBuffers,@@ -59,8 +53,7 @@ impl ClassifierContext {let mtl_command_queue =mtl_device.new_command_queue_with_max_command_buffer_count(1024);- let command_buffer =- MPSCommandBuffer::from_command_queue(&mtl_command_queue);+ let command_buffer = mtl_command_queue.new_command_buffer().to_owned();let config_path = model_path.join("config.json");if !config_path.exists() {@@ -137,12 +130,8 @@ impl ClassifierContext {.activation_precision().into();- let embed = embed_block(- &decoder_config,- &mtl_context,- &compilation_config.descriptor_general,- &root_loader_view,- );+ let embed =+ embed_block(&decoder_config, &mtl_context, &root_loader_view);let global_rope =Self::create_rope_block(&mtl_context, data_type, RopeType::Global)@@ -315,50 +304,22 @@ impl ClassifierContext {&prediction_head_dense_tree,ArrayId::ClassifierPooling,ArrayId::ClassifierPredictionHeadDense,- &compilation_config.descriptor_general,);- let prediction_head_activation = autoreleasepool(|_| {- let graph = Graph::new();- let input_shape = [-1, model_dim as isize];- let input_placeholder =- placeholder(&graph, &input_shape, prediction_head_data_type);-- let output = activation(- &graph,- &prediction_head_config.activation,- &input_placeholder,+ let prediction_head_activation = Box::new(+ Activation::new(+ &mtl_context,prediction_head_data_type,- );-- let shaped_type_obj =- shaped_type(&input_shape, prediction_head_data_type);- let feeds =- HashMap::from([(&*input_placeholder, &*shaped_type_obj)]);-- let executable = graph.compile(- &MPSDevice::with_device(&mtl_context.device),- &feeds,- &[&output],- None,- Some(&compilation_config.descriptor_general),- );-- let arguments = IOArrays::new(- vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),- vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),- );-- let execution_descriptor =- mpsgraph::ExecutableExecutionDescriptor::new();- execution_descriptor.set_enable_commit_and_continue(true);-- Box::new(MPSGraphBlock::new(- executable,- execution_descriptor,- arguments,- )) as Box<dyn EncodableBlock>- });+ prediction_head_config.activation.clone(),+ ArrayId::ClassifierPredictionHeadDense,+ ArrayId::ClassifierPredictionHeadDense,+ )+ .map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(+ format!("prediction head activation: {:?}", e),+ ))+ })?,+ );let prediction_head_norm_tree =prediction_head_tree.subtree("norm").map_err(|_| {@@ -396,7 +357,6 @@ impl ClassifierContext {&prediction_head_readout_tree,ArrayId::ClassifierPredictionHeadNorm,ArrayId::ClassifierPredictionHeadLogits,- &compilation_config.descriptor_general,).map_err(|e| {Error::Classifier(ClassifierError::KernelCreationFailed(format!(@@ -474,8 +434,7 @@ impl ClassifierContext {}pub fn reset_command_buffer(&mut self) {- self.command_buffer = MPSCommandBuffer::from_command_queue(- &self.mtl_context.command_queue,- );+ self.command_buffer =+ self.mtl_context.command_queue.new_command_buffer().to_owned();}}diff --git a/crates/uzu/src/backends/metal/compilation_parameters.rs b/crates/uzu/src/backends/metal/compilation_parameters.rsindex 3674ea12b..1bae0ea58 100644--- a/crates/uzu/src/backends/metal/compilation_parameters.rs+++ b/crates/uzu/src/backends/metal/compilation_parameters.rs@@ -1,73 +1,8 @@-use mpsgraph::{- CompilationDescriptor, ComputeDevice, Optimization, OptimizationProfile,-};-use objc2::rc::Retained;-#[derive(Debug, Clone, Copy)]-pub enum BlockDevice {- Ane,- Gpu,-}--pub fn make_compilation_descriptor(- device: BlockDevice,- optimization_level: Optimization,- optimization_profile: OptimizationProfile,- preform_placement_analysis: bool,-) -> Retained<CompilationDescriptor> {- let desc = CompilationDescriptor::new();- desc.set_optimization_level(optimization_level);- desc.set_optimization_profile(optimization_profile);- desc.set_print_ane_placement_analysis(preform_placement_analysis);- match device {- BlockDevice::Ane => {- desc.set_preferred_device(ComputeDevice::ANE);- desc.set_allowed_compute_devices(- ComputeDevice::ANE | ComputeDevice::GPU,- );- },- BlockDevice::Gpu => {- desc.set_preferred_device(ComputeDevice::GPU);- desc.set_allowed_compute_devices(ComputeDevice::GPU);- },- }- desc-}--pub struct CompilationConfig {- pub descriptor_general: Retained<CompilationDescriptor>,- pub descriptor_mlp: Retained<CompilationDescriptor>,-}+pub struct CompilationConfig {}impl CompilationConfig {- fn new(- descriptor_general: Retained<CompilationDescriptor>,- descriptor_mlp: Retained<CompilationDescriptor>,- ) -> Self {- Self {- descriptor_general,- descriptor_mlp,- }- }-pub fn default() -> Self {- let optimization_level = Optimization::Level1;- let optimization_profile = OptimizationProfile::Performance;- let perform_placement_analysis = false;-- Self::new(- make_compilation_descriptor(- BlockDevice::Gpu,- optimization_level,- optimization_profile,- perform_placement_analysis,- ),- make_compilation_descriptor(- BlockDevice::Gpu,- optimization_level,- optimization_profile,- perform_placement_analysis,- ),- )+ Self {}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rsnew file mode 100644index 000000000..e878e9ce1--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs@@ -0,0 +1,86 @@+use metal::{CommandBufferRef, ComputeCommandEncoderRef};++use super::{+ super::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::activation::ActivationKernel,+ },+ EncodableBlock, EncodingParameters,+};+use crate::{Array, DataType, config::Activation as ActivationConfig};++pub struct Activation {+ kernel: ActivationKernel,+ config: ActivationConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+}++impl Activation {+ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ config: ActivationConfig,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, MTLError> {+ let kernel = ActivationKernel::new(context, data_type)?;+ Ok(Self {+ kernel,+ config,+ input_array_id,+ output_array_id,+ })+ }+}++impl EncodableBlock for Activation {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBufferRef,+ _parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let mut input_array = arrays[0].borrow_mut();+ let mut output_array = arrays[1].borrow_mut();++ let n = input_array.shape().iter().product();++ let input_buffer = unsafe { input_array.mtl_buffer() };+ let output_buffer = unsafe { output_array.mtl_buffer() };++ let encoder = command_buffer.new_compute_command_encoder();++ self.kernel+ .encode(encoder, &self.config, input_buffer, output_buffer, n)+ .expect("Failed to encode activation kernel");++ encoder.end_encoding();+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ _parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let mut input_array = arrays[0].borrow_mut();+ let mut output_array = arrays[1].borrow_mut();++ let n = input_array.shape().iter().product();++ let input_buffer = unsafe { input_array.mtl_buffer() };+ let output_buffer = unsafe { output_array.mtl_buffer() };++ self.kernel+ .encode(encoder, &self.config, input_buffer, output_buffer, n)+ .expect("Failed to encode activation kernel");+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rsindex 91137f568..226465876 100644--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs@@ -1,6 +1,6 @@//! Attention kernel encodable.-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters};use crate::backends::metal::{@@ -48,7 +48,7 @@ impl EncodableBlock for Attention {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let (@@ -145,9 +145,7 @@ impl EncodableBlock for Attention {let qkv_buffer = unsafe { qkv_array.mtl_buffer() };// Prepare command encoder early (used for optional value extraction)- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();+ let compute_encoder = command_buffer.new_compute_command_encoder();// Get KV cache buffers only if KV cache exists (LLM mode)let has_kv_cache = state.cache_layers().is_some();@@ -331,10 +329,5 @@ impl EncodableBlock for Attention {}compute_encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }}}diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsindex fb9ab8e9f..298cb7849 100644--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs@@ -1,6 +1,6 @@use std::rc::Rc;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use objc2::rc::autoreleasepool;use super::{@@ -40,7 +40,7 @@ impl ClassifierLayer {pub fn new(mtl_context: &MTLContext,layer_config: &TransformerLayerConfig,- compilation_config: Rc<CompilationConfig>,+ _compilation_config: Rc<CompilationConfig>,layer_index: usize,model_dim: usize,hidden_dim: usize,@@ -112,7 +112,6 @@ impl ClassifierLayer {&layer_loader.subtree("mixer.qkv_projection").unwrap(),ArrayId::Main,ArrayId::QKV,- &compilation_config.descriptor_mlp,).expect("Failed to create qkv projection");@@ -151,7 +150,6 @@ impl ClassifierLayer {&layer_loader.subtree("mixer.out_projection").unwrap(),ArrayId::AttentionOutput,ArrayId::Main,- &compilation_config.descriptor_mlp,).expect("Failed to create out projection");@@ -210,7 +208,6 @@ impl ClassifierLayer {hidden_dim,mtl_context,&layer_loader.subtree("mlp").unwrap(),- &compilation_config.descriptor_mlp,).expect("Failed to create mlp block");@@ -278,7 +275,7 @@ impl EncodableBlock for ClassifierLayer {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {#[cfg(feature = "tracing")]diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rsindex c05f7b2f3..481ca2559 100644--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs@@ -2,7 +2,7 @@use std::rc::Rc;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,@@ -40,19 +40,11 @@ impl Decoder {.subtree("transformer").expect("transformer subtree not found");- let embed = embed_block(- &decoder_config,- &mtl_context,- &compilation_config.descriptor_general,- root_weight_loader,- );+ let embed =+ embed_block(&decoder_config, &mtl_context, root_weight_loader);- let readout = readout_block(- &decoder_config,- &mtl_context,- &compilation_config.descriptor_general,- root_weight_loader,- );+ let readout =+ readout_block(&decoder_config, &mtl_context, root_weight_loader);let attention_data_type = Self::attention_data_type(&decoder_config);let norm_reference_layer = decoder_config@@ -209,7 +201,7 @@ impl EncodableBlock for Decoder {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let mut resolver = EncoderResolver::new(command_buffer);diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rsindex c13db8c9a..da85f713d 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs@@ -1,7 +1,7 @@use crate::{DataType, backends::metal::MTLError};#[derive(Debug, thiserror::Error)]-pub enum QuantizedEmbeddingError {+pub enum EmbeddingError {#[error("Metal error: {0}")]MetalError(#[from] MTLError),#[error("Unsupported data type: {0:?}")]diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rsindex 5bd6c6ae1..527c1febc 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs@@ -1,9 +1,11 @@use std::rc::Rc;-use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef};-use super::QuantizedEmbeddingError;+use super::{+ super::{EncodableBlock, EncodingParameters},+ EmbeddingError,+};use crate::{Array, DataType,backends::metal::{@@ -17,8 +19,6 @@ use crate::{parameters::ParameterTree,};-use super::super::{EncodableBlock, EncodingParameters};-pub struct FullPrecisionEmbeddingLookup {kernel: FullPrecisionEmbeddingLookupKernel,weights_buffer: MTLBuffer,@@ -35,14 +35,14 @@ impl FullPrecisionEmbeddingLookup {model_dim: usize,input_scale: Option<f32>,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {let kernel =FullPrecisionEmbeddingLookupKernel::new(mtl_context, data_type)?;let mut weights = match parameter_tree.leaf("weights") {Ok(weights) => weights,Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}",e)))@@ -50,7 +50,7 @@ impl FullPrecisionEmbeddingLookup {};if weights.shape() != [vocab_size, model_dim] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding lookup weights shape mismatch: got {:?}, \expected [{}, {}]",@@ -62,7 +62,7 @@ impl FullPrecisionEmbeddingLookup {}if weights.data_type() != data_type {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Weights dtype mismatch: got {:?}, expected {:?}",weights.data_type(),@@ -87,7 +87,7 @@ impl EncodableBlock for FullPrecisionEmbeddingLookup {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &CommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);@@ -98,8 +98,7 @@ impl EncodableBlock for FullPrecisionEmbeddingLookup {let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let args = FullPrecisionEmbeddingLookupArguments {token_ids_buffer,@@ -118,11 +117,8 @@ impl EncodableBlock for FullPrecisionEmbeddingLookup {encoder.end_encoding();if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}-diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rsindex b3c6c0860..9d373f2ca 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs@@ -1,11 +1,10 @@use std::{cell::RefCell, rc::Rc};-use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef};use super::{super::{EncodableBlock, EncodingParameters},- QuantizedEmbeddingError,+ EmbeddingError,};use crate::{Array, DataType,@@ -31,10 +30,10 @@ impl FullPrecisionEmbeddingReadout {vocab_size: usize,model_dim: usize,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32){- return Err(QuantizedEmbeddingError::UnsupportedDataType(+ return Err(EmbeddingError::UnsupportedDataType(data_type,));}@@ -42,7 +41,7 @@ impl FullPrecisionEmbeddingReadout {let mut weights = match parameter_tree.leaf("weights") {Ok(weights) => weights,Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}",e)))@@ -50,7 +49,7 @@ impl FullPrecisionEmbeddingReadout {};if weights.shape() != [vocab_size, model_dim] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",weights.shape(),@@ -61,7 +60,7 @@ impl FullPrecisionEmbeddingReadout {}if weights.data_type() != data_type {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Weights dtype mismatch: got {:?}, expected {:?}",weights.data_type(),@@ -75,7 +74,7 @@ impl FullPrecisionEmbeddingReadout {// Weights are [vocab_size, model_dim], we compute input @ weights^T// MatmulKernel with transpose_b=true handles thislet kernel = MatmulKernel::new(mtl_context, data_type, false, true)- .map_err(QuantizedEmbeddingError::MetalError)?;+ .map_err(EmbeddingError::MetalError)?;Ok(Self {kernel: RefCell::new(kernel),@@ -90,7 +89,7 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &CommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);@@ -101,8 +100,7 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {let input_buffer = unsafe { input_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let args = MatmulArguments {a: input_buffer,@@ -125,10 +123,8 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {encoder.end_encoding();if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rsindex 3c1386abd..e376c571b 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs@@ -6,7 +6,7 @@ mod full_precision_readout;mod quantized_lookup;mod quantized_readout;-pub use error::QuantizedEmbeddingError;+pub use error::EmbeddingError;pub use full_precision_lookup::FullPrecisionEmbeddingLookup;pub use full_precision_readout::FullPrecisionEmbeddingReadout;pub use quantized_lookup::QuantizedEmbeddingLookup;diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rsindex 4c1510b2a..9f4099dc3 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs@@ -1,11 +1,10 @@use std::rc::Rc;-use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef};use super::{super::{EncodableBlock, EncodingParameters},- QuantizedEmbeddingError,+ EmbeddingError,};use crate::{Array, DataType,@@ -39,7 +38,7 @@ impl QuantizedEmbeddingLookup {group_size: usize,mode: QuantizationMode,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {Self::new_with_names(mtl_context,data_type,@@ -62,7 +61,7 @@ impl QuantizedEmbeddingLookup {group_size: usize,mode: QuantizationMode,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {Self::new_with_names(mtl_context,data_type,@@ -88,7 +87,7 @@ impl QuantizedEmbeddingLookup {scales_name: &str,biases_name: &str,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {let packing_divisor = mode.packing_divisor();let kernel =@@ -96,14 +95,14 @@ impl QuantizedEmbeddingLookup {// Load weights [vocab_size, model_dim/packing_divisor] as storage_typelet mut weights = parameter_tree.leaf(weights_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}",e)))})?;if weights.data_type() != mode.storage_type() {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Expected packed weights of type {:?}, got {:?}",mode.storage_type(),@@ -114,7 +113,7 @@ impl QuantizedEmbeddingLookup {// Load scales [vocab_size, num_groups]let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load scales: {:?}",e)))@@ -123,7 +122,7 @@ impl QuantizedEmbeddingLookup {// Validate shapes and typeslet num_groups = (model_dim + group_size - 1) / group_size;if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",weights.shape(),@@ -133,7 +132,7 @@ impl QuantizedEmbeddingLookup {));}if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",scales.shape(),@@ -143,7 +142,7 @@ impl QuantizedEmbeddingLookup {));}if scales.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(+ return Err(EmbeddingError::UnsupportedDataType(scales.data_type(),));}@@ -152,7 +151,7 @@ impl QuantizedEmbeddingLookup {let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {Ok(mut deq_biases) => {if deq_biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",deq_biases.shape(),@@ -162,7 +161,7 @@ impl QuantizedEmbeddingLookup {));}if deq_biases.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(+ return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type(),));}@@ -174,7 +173,7 @@ impl QuantizedEmbeddingLookup {DataType::F32 => 4,other => {return Err(- QuantizedEmbeddingError::UnsupportedDataType(other),+ EmbeddingError::UnsupportedDataType(other),);},};@@ -213,7 +212,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &CommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);@@ -224,8 +223,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let args = QuantizedEmbeddingLookupArguments {token_ids_buffer,@@ -246,10 +244,8 @@ impl EncodableBlock for QuantizedEmbeddingLookup {encoder.end_encoding();if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rsindex 6b619ae93..064ed8ca7 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs@@ -1,11 +1,10 @@use std::rc::Rc;-use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef};use super::{super::{EncodableBlock, EncodingParameters},- QuantizedEmbeddingError,+ EmbeddingError,};use crate::{Array, DataType,@@ -38,7 +37,7 @@ impl QuantizedEmbeddingReadout {group_size: usize,mode: QuantizationMode,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {Self::new_with_names(mtl_context,data_type,@@ -61,7 +60,7 @@ impl QuantizedEmbeddingReadout {group_size: usize,mode: QuantizationMode,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {Self::new_with_names(mtl_context,data_type,@@ -87,10 +86,10 @@ impl QuantizedEmbeddingReadout {scales_name: &str,biases_name: &str,parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {// Load weights [vocab_size, model_dim/2] as U8let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}",e)))@@ -98,7 +97,7 @@ impl QuantizedEmbeddingReadout {// Load scales [vocab_size, num_groups]let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load scales: {:?}",e)))@@ -112,7 +111,7 @@ impl QuantizedEmbeddingReadout {let weights_transposed = weights.shape()[0] == vocab_size;if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",weights.shape(),@@ -122,7 +121,7 @@ impl QuantizedEmbeddingReadout {));}if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",scales.shape(),@@ -132,7 +131,7 @@ impl QuantizedEmbeddingReadout {));}if scales.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(+ return Err(EmbeddingError::UnsupportedDataType(scales.data_type(),));}@@ -141,7 +140,7 @@ impl QuantizedEmbeddingReadout {let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {Ok(mut deq_biases) => {if deq_biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(+ return Err(EmbeddingError::MetalError(MTLError::Generic(format!("Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",deq_biases.shape(),@@ -151,7 +150,7 @@ impl QuantizedEmbeddingReadout {));}if deq_biases.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(+ return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type(),));}@@ -164,7 +163,7 @@ impl QuantizedEmbeddingReadout {DataType::F32 => 4,other => {return Err(- QuantizedEmbeddingError::UnsupportedDataType(other),+ EmbeddingError::UnsupportedDataType(other),);},};@@ -198,7 +197,7 @@ impl QuantizedEmbeddingReadout {weights_transposed,).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ EmbeddingError::MetalError(MTLError::Generic(format!("Failed to create kernel: {:?}",e)))@@ -219,7 +218,7 @@ impl EncodableBlock for QuantizedEmbeddingReadout {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &CommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);@@ -230,8 +229,7 @@ impl EncodableBlock for QuantizedEmbeddingReadout {let input_buffer = unsafe { input_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let args = QuantizedMatmulArguments {a_buffer: input_buffer,@@ -252,10 +250,8 @@ impl EncodableBlock for QuantizedEmbeddingReadout {encoder.end_encoding();if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsindex a2cd5dc3a..70fdc92cc 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -2,7 +2,7 @@use std::rc::Rc;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use objc2::rc::autoreleasepool;use super::{@@ -114,7 +114,6 @@ impl LayerExecutables {.unwrap(),ArrayId::Main,ArrayId::QKV,- &compilation_config.descriptor_mlp,).expect("Failed to create qkv projection");@@ -155,7 +154,6 @@ impl LayerExecutables {.unwrap(),ArrayId::AttentionOutput,ArrayId::Main,- &compilation_config.descriptor_mlp,).expect("Failed to create out projection");@@ -263,7 +261,6 @@ impl LayerExecutables {hidden_dim,mtl_context,&decoder_layer_loader.subtree("mlp").unwrap(),- &compilation_config.descriptor_mlp,).expect("Failed to create mlp block");@@ -305,7 +302,7 @@ impl EncodableBlock for LayerExecutables {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {#[cfg(feature = "tracing")]diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsindex 155324388..c8bf57461 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs@@ -1,7 +1,6 @@use std::{cell::RefCell, rc::Rc};-use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef};use crate::{DataType,@@ -117,7 +116,7 @@ impl EncodableBlock for FullPrecisionLinear {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);@@ -129,8 +128,7 @@ impl EncodableBlock for FullPrecisionLinear {let input_buffer = unsafe { input_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let args = MatmulArguments {a: input_buffer,@@ -156,24 +154,16 @@ impl EncodableBlock for FullPrecisionLinear {(&self.bias_add_kernel, &self.biases_buffer){let total_len = batch_size * self.output_dim;- let retained_cb = root_command_buffer.to_owned();bias_add.encode_into_command_buffer(&output_buffer,bias_buf,&output_buffer,self.output_dim,total_len,- &retained_cb,+ command_buffer,parameters.predicate,);}-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }}fn supports_shared_encoder(&self) -> bool {diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rsindex 9fab0314e..1027bab77 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs@@ -1,7 +1,6 @@use std::rc::Rc;-use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef};use crate::{DataType,@@ -226,7 +225,7 @@ impl EncodableBlock for QuantizedLinear {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);@@ -238,8 +237,7 @@ impl EncodableBlock for QuantizedLinear {let input_buffer = unsafe { input_array_mut.mtl_buffer() };let output_buffer = unsafe { output_array_mut.mtl_buffer() };- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let args = QuantizedMatmulArguments {a_buffer: input_buffer,@@ -263,23 +261,20 @@ impl EncodableBlock for QuantizedLinear {(&self.bias_add_kernel, &self.biases_buffer){let total_len = batch_size * self.output_dim;- let retained_cb = root_command_buffer.to_owned();bias_add.encode_into_command_buffer(&output_buffer,bias_buf,&output_buffer,self.output_dim,total_len,- &retained_cb,+ command_buffer,parameters.predicate,);}if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsindex 047656f42..d2e80789f 100644--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs@@ -2,7 +2,7 @@use std::{env, rc::Rc};-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters, transformer_layer};use crate::{@@ -44,7 +44,7 @@ impl MambaMixer {mtl_context: &MTLContext,layer_type: DecoderLayerType,mamba_config: Mamba2Config,- compilation_config: Rc<CompilationConfig>,+ _compilation_config: Rc<CompilationConfig>,layer_index: usize,model_dim: usize,num_heads: usize,@@ -82,7 +82,6 @@ impl MambaMixer {),ArrayId::Main,ArrayId::SsmInProj,- &compilation_config.descriptor_mlp,).expect("Failed to create in-projection kernel");@@ -98,7 +97,6 @@ impl MambaMixer {),ArrayId::AttentionOutput,ArrayId::Main,- &compilation_config.descriptor_mlp,).expect("Failed to create out-projection kernel");@@ -147,7 +145,7 @@ impl MambaMixer {fn encode_pipeline(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let active_suffix_length = state.active_suffix_length();@@ -168,17 +166,15 @@ impl MambaMixer {self.out_projection.encode(state, command_buffer, parameters);if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}fn run_split_inproj(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,suffix_length: usize,) {let arrays = state.arrays(&[@@ -204,9 +200,7 @@ impl MambaMixer {let num_heads = self.config.num_heads;let total_dim = conv_dim + inner_dim + num_heads;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute = mtl_command_buffer.new_compute_command_encoder();+ let compute = command_buffer.new_compute_command_encoder();self.split_inproj.encode(&compute,@@ -230,7 +224,7 @@ impl MambaMixer {fn run_conv_scan(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,suffix_length: usize,) {let arrays = state.arrays(&[@@ -262,7 +256,7 @@ impl MambaMixer {let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let proj_dim = self.config.num_groups * self.config.state_dim;- let cmd = command_buffer.root_command_buffer().to_owned();+ let cmd = command_buffer.to_owned();let state_stride = self.config.kernel_size.saturating_sub(1);drop(conv_state);@@ -355,7 +349,7 @@ impl MambaMixer {fn run_prefill_ssm(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,suffix_length: usize,) {let base_arrays = state.arrays(&[@@ -392,7 +386,7 @@ impl MambaMixer {let mut skip_weights = self.skip_connection_weight.clone();let skip = unsafe { skip_weights.mtl_buffer().to_owned() };- let cmd = command_buffer.root_command_buffer().to_owned();+ let cmd = command_buffer.to_owned();let compute = cmd.new_compute_command_encoder();self.ssm_prefill.encode(@@ -438,7 +432,7 @@ impl MambaMixer {fn run_decode_ssm(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,suffix_length: usize,) {let arrays = state.arrays(&[@@ -485,7 +479,7 @@ impl MambaMixer {let group_size = (h / g) as i32;let state_size = n as i32;- let cmd = command_buffer.root_command_buffer().to_owned();+ let cmd = command_buffer.to_owned();let compute = cmd.new_compute_command_encoder();self.ssd_update.encode(@@ -533,7 +527,7 @@ impl EncodableBlock for MambaMixer {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {self.encode_pipeline(state, command_buffer, parameters);diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex 3b9c104fe..e3cbf1a28 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -1,6 +1,6 @@//! MLP block encodable.-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters, QuantizedLinear};use crate::backends::metal::{@@ -32,7 +32,7 @@ impl EncodableBlock for MlpBlock {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,params: &EncodingParameters,) {// Up@@ -42,8 +42,7 @@ impl EncodableBlock for MlpBlock {let mut fused = arrays[0].borrow_mut();let mut hidden = arrays[1].borrow_mut();let active_suffix_length = state.active_suffix_length() as i32;- let root = command_buffer.root_command_buffer();- let encoder = root.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let fused_buf = unsafe { fused.mtl_buffer() };let hidden_buf = unsafe { hidden.mtl_buffer() };self.gate@@ -56,10 +55,8 @@ impl EncodableBlock for MlpBlock {self.down.encode(state, command_buffer, params);if params.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 43ff39948..ea80d2c61 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -6,11 +6,11 @@//! Encodables implement `EncodableBlock` and orchestrate one or more//! kernels to perform operations on `ForwardPassState`.-use metal::ComputeCommandEncoderRef;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::{CommandBufferRef, ComputeCommandEncoderRef};use super::forward_pass::ForwardPassState;+mod activation;mod attention;mod classifier_layer;mod decoder;@@ -31,12 +31,14 @@ mod tensor_add_swap;mod tensor_copy;pub mod transformer_layer;+pub use activation::Activation;pub use attention::Attention;pub use classifier_layer::ClassifierLayer;pub use decoder::Decoder;pub use embedding::{- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout,- QuantizedEmbeddingError, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,+ EmbeddingError, FullPrecisionEmbeddingLookup,+ FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,+ QuantizedEmbeddingReadout,};pub use encoding_parameters::EncodingParameters;pub use layer::LayerExecutables;@@ -60,7 +62,7 @@ pub trait EncodableBlock {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,);diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsindex ec45d94b4..35b9ec7c8 100644--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs@@ -2,7 +2,7 @@use std::{cell::RefCell, rc::Rc};-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{super::{EncodableBlock, EncodingParameters},@@ -291,7 +291,7 @@ impl EncodableBlock for MoeBlock {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let suffix_length = state.active_suffix_length();@@ -350,8 +350,7 @@ impl EncodableBlock for MoeBlock {let e = self.moe_config.mixture_size;let k = self.moe_config.num_experts_per_token;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();+ let mtl_command_buffer = command_buffer.to_owned();let root = &*mtl_command_buffer;let k_tile = 128;@@ -626,8 +625,8 @@ impl EncodableBlock for MoeBlock {.expect("MoE finalize failed");if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rsindex a9e2b56a5..af9997007 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs@@ -2,8 +2,7 @@use std::rc::Rc;-use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef};use super::super::{EncodableBlock, EncodingParameters};use crate::{@@ -89,7 +88,7 @@ impl EncodableBlock for LayerNorm {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let input_binding = state.arrays(&[self.input_array_id]);@@ -109,9 +108,7 @@ impl EncodableBlock for LayerNorm {let batch_size = input_shape[0] as i32;let model_dim = input_shape[1] as i32;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();+ let compute_encoder = command_buffer.new_compute_command_encoder();if let Err(e) = self.kernel.encode(&compute_encoder,@@ -131,8 +128,8 @@ impl EncodableBlock for LayerNorm {compute_encoder.end_encoding();if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rsindex d2d7309cf..01c3e6ff5 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs@@ -7,7 +7,7 @@ mod rms_norm;use std::rc::Rc;pub use layer_norm::LayerNorm;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;pub use qk_norm::QKNorm;pub use rms_norm::RMSNorm;@@ -77,7 +77,7 @@ impl EncodableBlock for Normalization {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {match self {diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rsindex c8e39f808..8d7f9f7cc 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs@@ -2,8 +2,7 @@use std::rc::Rc;-use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef};use super::super::{EncodableBlock, EncodingParameters};use crate::{@@ -163,7 +162,7 @@ impl EncodableBlock for QKNorm {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let qkv_binding = state.arrays(&[self.qkv_array_id]);@@ -178,9 +177,7 @@ impl EncodableBlock for QKNorm {let batch_size = qkv_shape[0] as i32;let head_dim = self.head_dim as i32;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();+ let compute_encoder = command_buffer.new_compute_command_encoder();// Process query normalization if configuredif let (@@ -240,8 +237,8 @@ impl EncodableBlock for QKNorm {compute_encoder.end_encoding();if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rsindex 83a96877e..d1ea6d1cb 100644--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs@@ -2,8 +2,7 @@use std::rc::Rc;-use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::{Buffer as MTLBuffer, CommandBufferRef};use super::super::{EncodableBlock, EncodingParameters};use crate::{@@ -93,7 +92,7 @@ impl EncodableBlock for RMSNorm {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let input_binding = state.arrays(&[self.input_array_id]);@@ -113,9 +112,7 @@ impl EncodableBlock for RMSNorm {let batch_size = input_shape[0] as i32;let model_dim = input_shape[1] as i32;- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();+ let compute_encoder = command_buffer.new_compute_command_encoder();if let Err(e) = self.kernel.encode(&compute_encoder,@@ -135,8 +132,8 @@ impl EncodableBlock for RMSNorm {compute_encoder.end_encoding();if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rsindex 323585f66..deb420f0b 100644--- a/crates/uzu/src/backends/metal/encodable_block/pooling.rs+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs@@ -1,6 +1,6 @@//! Pooling encodable for sequence-level aggregation.-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters};use crate::{@@ -36,7 +36,7 @@ impl EncodableBlock for Pooling {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,_parameters: &EncodingParameters,) {let batch_size = 1;@@ -51,8 +51,7 @@ impl EncodableBlock for Pooling {let mut pooling_array = arrays[1].borrow_mut();let output_buffer = unsafe { pooling_array.mtl_buffer().to_owned() };- let root = command_buffer.root_command_buffer();- let encoder = root.new_compute_command_encoder();+ let encoder = command_buffer.new_compute_command_encoder();let result = match self.pooling_type {PoolingType::Cls => self.pooling_kernel.encode_cls(@@ -81,8 +80,6 @@ impl EncodableBlock for Pooling {drop(main_array);drop(pooling_array);- command_buffer.commit_and_continue();-#[cfg(feature = "tracing")]{let traces_rc = state.traces().clone();@@ -92,8 +89,7 @@ impl EncodableBlock for Pooling {drop(trace_arr);drop(traces_ref);- let root = command_buffer.root_command_buffer();- let blit = root.new_blit_command_encoder();+ let blit = command_buffer.new_blit_command_encoder();blit.copy_from_buffer(&output_buffer,0,@@ -103,10 +99,6 @@ impl EncodableBlock for Pooling {as u64,);blit.end_encoding();-- let root_owned = root.to_owned();- command_buffer.commit_and_continue();- root_owned.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rsindex 25a4fcf4f..3f468746a 100644--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs@@ -1,6 +1,6 @@//! Prediction head encodable for classification output.-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters};#[cfg(feature = "tracing")]@@ -40,7 +40,7 @@ impl EncodableBlock for ClassifierPredictionHead {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {self.dense.encode(state, command_buffer, parameters);@@ -49,11 +49,15 @@ impl EncodableBlock for ClassifierPredictionHead {self.readout.encode(state, command_buffer, parameters);#[cfg_attr(feature = "tracing", allow(unused_variables))]- let root_after_head = command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();+ let root_after_head = command_buffer.to_owned();+ if parameters.wait_until_completed {+ command_buffer.commit();+ }#[cfg(not(feature = "tracing"))]- root_after_head.wait_until_completed();+ if parameters.wait_until_completed {+ root_after_head.wait_until_completed();+ }#[cfg(feature = "tracing")]{@@ -77,8 +81,7 @@ impl EncodableBlock for ClassifierPredictionHead {(batch_size * self.num_labels * data_type.size_in_bytes())as u64;- let root = command_buffer.root_command_buffer();- let blit = root.new_blit_command_encoder();+ let blit = command_buffer.new_blit_command_encoder();blit.copy_from_buffer(&linear_output_buffer,0,@@ -88,8 +91,8 @@ impl EncodableBlock for ClassifierPredictionHead {);blit.end_encoding();- let root_owned = root.to_owned();- command_buffer.commit_and_continue();+ let root_owned = command_buffer.to_owned();+ command_buffer.commit();root_owned.wait_until_completed();}}diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rsindex 94b8731da..4163fa962 100644--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs@@ -1,6 +1,6 @@//! Rope (Rotary Position Embedding) encodable.-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters};use crate::{@@ -35,7 +35,7 @@ impl EncodableBlock for Rope {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {@@ -81,9 +81,7 @@ impl EncodableBlock for Rope {state.arrays(&[ArrayId::RopeSines(self.rope_type)]);let mut rope_sines = sin_buffer_binding[0].borrow_mut();- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();+ let compute_encoder = command_buffer.new_compute_command_encoder();let token_positions_offset = token_positions.buffer_offset();@@ -112,8 +110,8 @@ impl EncodableBlock for Rope {compute_encoder.end_encoding();if parameters.wait_until_completed {- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/sampling.rs b/crates/uzu/src/backends/metal/encodable_block/sampling.rsindex 7b1d21648..50a3ab92f 100644--- a/crates/uzu/src/backends/metal/encodable_block/sampling.rs+++ b/crates/uzu/src/backends/metal/encodable_block/sampling.rs@@ -2,7 +2,7 @@use std::rc::Rc;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters};use crate::backends::metal::{@@ -57,7 +57,7 @@ impl EncodableBlock for Sampling {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {assert!(@@ -84,8 +84,7 @@ impl EncodableBlock for Sampling {let sampling_method = state.sampling_method().unwrap();let seeds_offset = seeds.buffer_offset();- let root_command_buffer =- command_buffer.root_command_buffer().to_owned();+ let root_command_buffer = command_buffer.to_owned();let bitmask_buffer = state.token_bitmask().map(|cell| unsafe { cell.borrow_mut().mtl_buffer().clone() });@@ -104,8 +103,8 @@ impl EncodableBlock for Sampling {}if parameters.wait_until_completed {- command_buffer.commit_and_continue();- root_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex 8823cf39b..f3c280e2e 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -1,6 +1,6 @@use std::rc::Rc;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters, transformer_layer};use crate::{@@ -47,7 +47,7 @@ impl ShortConvMixer {mtl_context: &MTLContext,layer_type: DecoderLayerType,short_conv_config: ShortConvConfig,- compilation_config: Rc<CompilationConfig>,+ _compilation_config: Rc<CompilationConfig>,layer_index: usize,model_dim: usize,decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,@@ -77,7 +77,6 @@ impl ShortConvMixer {&resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),ArrayId::Main,ArrayId::SsmInProj,- &compilation_config.descriptor_mlp,).expect("Failed to create in-projection kernel");@@ -90,7 +89,6 @@ impl ShortConvMixer {&resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),ArrayId::AttentionOutput,ArrayId::Main,- &compilation_config.descriptor_mlp,).expect("Failed to create out-projection kernel");@@ -120,7 +118,7 @@ impl ShortConvMixer {fn encode_pipeline(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let active_suffix_length = state.active_suffix_length();@@ -139,17 +137,15 @@ impl ShortConvMixer {self.out_projection.encode(state, command_buffer, parameters);if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}fn run_prefill_conv(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,suffix_length: usize,) {let padded = state@@ -237,7 +233,7 @@ impl ShortConvMixer {fn run_decode_conv(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,suffix_length: usize,) {let arrays = state.arrays(&[@@ -263,9 +259,7 @@ impl ShortConvMixer {let kernel_size = self.config.kernel_size;let state_stride = kernel_size.saturating_sub(1);- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute = mtl_command_buffer.new_compute_command_encoder();+ let compute = command_buffer.new_compute_command_encoder();self.short_conv_kernel.encode_decode(@@ -294,7 +288,7 @@ impl EncodableBlock for ShortConvMixer {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {self.encode_pipeline(state, command_buffer, parameters);diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rsindex ae72e7d47..ba3837b22 100644--- a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs@@ -1,6 +1,6 @@//! Tensor add-swap encodable.-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters};use crate::{@@ -35,7 +35,7 @@ impl EncodableBlock for TensorAddSwap {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&self.argument_arrays);@@ -48,19 +48,16 @@ impl EncodableBlock for TensorAddSwap {let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };let main_mtl_buffer = unsafe { main_array.mtl_buffer() };- let retained_mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();-self.kernel.encode_into_command_buffer(&skip_mtl_buffer,&main_mtl_buffer,length,- &retained_mtl_command_buffer,+ command_buffer,);if parameters.wait_until_completed {- command_buffer.commit_and_continue();- retained_mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rsindex 61156c8fa..c2a090fed 100644--- a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs@@ -1,6 +1,6 @@//! Tensor copy encodable.-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBufferRef;use super::{EncodableBlock, EncodingParameters};use crate::{@@ -35,7 +35,7 @@ impl EncodableBlock for TensorCopy {fn encode(&self,state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,parameters: &EncodingParameters,) {let arrays = state.arrays(&self.argument_arrays);@@ -48,19 +48,16 @@ impl EncodableBlock for TensorCopy {let source_mtl_buffer = unsafe { source_array.mtl_buffer() };let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };- let retained_mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();-self.kernel.encode_into_command_buffer(&source_mtl_buffer,&destination_mtl_buffer,length,- &retained_mtl_command_buffer,+ command_buffer,);if parameters.wait_until_completed {- command_buffer.commit_and_continue();- retained_mtl_command_buffer.wait_until_completed();+ command_buffer.commit();+ command_buffer.wait_until_completed();}}}diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 27d00f4c1..d26cf38b1 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -1,10 +1,4 @@-use std::{collections::HashMap, rc::Rc};--use mpsgraph::{- CompilationDescriptor, Device as MPSDevice, ExecutableExecutionDescriptor,- Graph,-};-use objc2::rc::Retained;+use std::rc::Rc;use super::{EncodableBlock, FullPrecisionEmbeddingLookup,@@ -14,24 +8,13 @@ use super::{use crate::{DataType,backends::metal::{- MTLContext, MTLError,- forward_pass::{ArrayId, IOArrays, MPSGraphBlock},- graph::{- embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,- embeddings_readout_subgraph, placeholder, shaped_type,- },+ MTLContext, MTLError, forward_pass::ArrayId,kernel::mlp::MlpGateActMulEncodable,},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},parameters::ParameterTree,};-fn make_execution_descriptor() -> Retained<ExecutableExecutionDescriptor> {- let execution_descriptor = ExecutableExecutionDescriptor::new();- execution_descriptor.set_enable_commit_and_continue(true);- execution_descriptor-}-pub fn linear_block<const N: usize>(config: &LinearConfig,_has_biases: bool,@@ -41,7 +24,6 @@ pub fn linear_block<const N: usize>(parameter_tree: &ParameterTree<Rc<MTLContext>>,input_array_id: ArrayId,output_array_id: ArrayId,- _compilation_descriptor: &CompilationDescriptor,) -> Result<Box<dyn EncodableBlock>, MTLError> {let out_sum: usize = output_dims.iter().sum();match config {@@ -86,7 +68,6 @@ pub fn mlp_block(hidden_dim: usize,context: &MTLContext,parameter_tree: &ParameterTree<Rc<MTLContext>>,- _compilation_descriptor: &CompilationDescriptor,) -> Result<Box<dyn EncodableBlock>, MTLError> {if let crate::config::MLPConfig::Dense(dense) = config {match &dense.linear_config {@@ -190,7 +171,6 @@ pub fn mlp_block(pub fn embed_block(config: &DecoderConfig,context: &MTLContext,- compilation_descriptor: &CompilationDescriptor,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Box<dyn EncodableBlock> {match &config.embedding_config {@@ -281,74 +261,29 @@ pub fn embed_block(Box::new(block)},EmbeddingConfig::QuantizedTied {- embedding_quantization_mode: _,+ embedding_quantization_mode,+ activation_precision,..} => {- let graph = Graph::new();-- let input_shape = [-1 as isize];- let input_data_type = DataType::U64;- let input_placeholder =- placeholder(&graph, &input_shape, input_data_type);- let input_shaped_type = shaped_type(&input_shape, input_data_type);- let weights_shape =- [config.vocab_size as isize, (config.model_dim / 2) as isize];- let weights_data_type = DataType::U8;- let weights_shaped_type =- shaped_type(&weights_shape, weights_data_type);- let weights_placeholder =- placeholder(&graph, &weights_shape, weights_data_type);-- let scales_shape = [config.vocab_size as isize];- let scales_data_type: DataType =- config.output_norm_config.scale_precision.into();- let scales_shaped_type =- shaped_type(&scales_shape, scales_data_type);- let scales_placeholder =- placeholder(&graph, &scales_shape, scales_data_type);-- let weights = embeddings_dequantize_weights_subgraph(- &graph,- &weights_placeholder,- &scales_placeholder,- )- .unwrap();+ let data_type: DataType = (*activation_precision).into();++ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");- let output = embeddings_embed_subgraph(- &graph,- &config.embedding_config,- &input_placeholder,- &weights,+ // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim+ let group_size = config.model_dim;++ let block = QuantizedEmbeddingLookup::new_tied(+ context,+ data_type,+ config.vocab_size,+ config.model_dim,+ group_size,+ *embedding_quantization_mode,+ &embeddings_tree,)- .unwrap();-- let feeds = HashMap::from([- (&*input_placeholder, &*input_shaped_type),- (&*weights_placeholder, &*weights_shaped_type),- (&*scales_placeholder, &*scales_shaped_type),- ]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(&compilation_descriptor),- );-- let block = MPSGraphBlock::new(- executable,- make_execution_descriptor(),- IOArrays::new(- vec![- ArrayId::TokenIds,- ArrayId::EmbeddingsInputWeights,- ArrayId::EmbeddingsScales,- ]- .into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- ),- );+ .expect("Failed to create quantized embedding lookup kernel");Box::new(block)},@@ -414,10 +349,9 @@ pub fn embed_block(pub fn readout_block(config: &DecoderConfig,context: &MTLContext,- compilation_descriptor: &CompilationDescriptor,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Box<dyn EncodableBlock> {- match config.embedding_config {+ match &config.embedding_config {EmbeddingConfig::Tied {precision,..@@ -428,7 +362,7 @@ pub fn readout_block(let block = FullPrecisionEmbeddingReadout::new(context,- precision.into(),+ (*precision).into(),config.vocab_size,config.model_dim,&embeddings_tree,@@ -447,7 +381,7 @@ pub fn readout_block(let block = FullPrecisionEmbeddingReadout::new(context,- precision.into(),+ (*precision).into(),config.vocab_size,config.model_dim,&embeddings_tree,@@ -462,7 +396,7 @@ pub fn readout_block(embedding_quantization_mode,..} => {- let data_type: DataType = activation_precision.into();+ let data_type: DataType = (*activation_precision).into();let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");@@ -472,8 +406,8 @@ pub fn readout_block(data_type,config.vocab_size,config.model_dim,- group_size,- embedding_quantization_mode,+ *group_size,+ *embedding_quantization_mode,&embeddings_tree,).expect("Failed to create quantized embedding readout kernel");@@ -481,75 +415,29 @@ pub fn readout_block(Box::new(block)},EmbeddingConfig::QuantizedTied {- embedding_quantization_mode: _,+ embedding_quantization_mode,+ activation_precision,..} => {- let graph = Graph::new();+ let data_type: DataType = (*activation_precision).into();- let input_shape = [-1 as isize, config.model_dim as isize];- let input_data_type: DataType =- config.output_norm_config.scale_precision.into();- let input_placeholder =- placeholder(&graph, &input_shape, input_data_type);- let input_shaped_type = shaped_type(&input_shape, input_data_type);-- let weights_shape =- [config.vocab_size as isize, (config.model_dim / 2) as isize];- let weights_data_type = DataType::U8;- let weights_shaped_type =- shaped_type(&weights_shape, weights_data_type);- let weights_placeholder =- placeholder(&graph, &weights_shape, weights_data_type);-- let scales_shape = [config.vocab_size as isize];- let scales_data_type: DataType =- config.output_norm_config.scale_precision.into();- let scales_shaped_type =- shaped_type(&scales_shape, scales_data_type);- let scales_placeholder =- placeholder(&graph, &scales_shape, scales_data_type);-- let weights = embeddings_dequantize_weights_subgraph(- &graph,- &weights_placeholder,- &scales_placeholder,- )- .unwrap();- let weights_transposed = graph.transpose(&weights, &[1, 0], None);- let output = embeddings_readout_subgraph(- &graph,- &input_placeholder,- &weights_transposed,+ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");++ // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim+ let group_size = config.model_dim;++ let block = QuantizedEmbeddingReadout::new_tied(+ context,+ data_type,+ config.vocab_size,+ config.model_dim,+ group_size,+ *embedding_quantization_mode,+ &embeddings_tree,)- .unwrap();-- let feeds = HashMap::from([- (&*input_placeholder, &*input_shaped_type),- (&*weights_placeholder, &*weights_shaped_type),- (&*scales_placeholder, &*scales_shaped_type),- ]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(compilation_descriptor),- );-- let block = MPSGraphBlock::new(- executable,- make_execution_descriptor(),- IOArrays::new(- vec![- ArrayId::Main,- ArrayId::EmbeddingsOutputWeights,- ArrayId::EmbeddingsScales,- ]- .into_boxed_slice(),- vec![ArrayId::Logits].into_boxed_slice(),- ),- );+ .expect("Failed to create quantized embedding readout kernel");Box::new(block)},@@ -570,8 +458,8 @@ pub fn readout_block(data_type,config.vocab_size,config.model_dim,- group_size,- embedding_quantization_mode,+ *group_size,+ *embedding_quantization_mode,&embeddings_tree,).expect("Failed to create quantized embedding readout kernel");diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rsindex d9724e1a4..ffa791574 100644--- a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs+++ b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs@@ -1,5 +1,4 @@-use metal::ComputeCommandEncoder;-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::{CommandBufferRef, ComputeCommandEncoder};use super::ForwardPassState;use crate::backends::metal::encodable_block::{@@ -7,12 +6,12 @@ use crate::backends::metal::encodable_block::{};pub struct EncoderResolver<'a> {- command_buffer: &'a MPSCommandBuffer,+ command_buffer: &'a CommandBufferRef,encoder: Option<ComputeCommandEncoder>,}impl<'a> EncoderResolver<'a> {- pub fn new(command_buffer: &'a MPSCommandBuffer) -> Self {+ pub fn new(command_buffer: &'a CommandBufferRef) -> Self {Self {command_buffer,encoder: None,@@ -29,7 +28,6 @@ impl<'a> EncoderResolver<'a> {if self.encoder.is_none() {self.encoder = Some(self.command_buffer- .root_command_buffer().new_compute_command_encoder().to_owned(),);diff --git a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs b/crates/uzu/src/backends/metal/forward_pass/io_arrays.rsdeleted file mode 100644index e8e7322e9..000000000--- a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs+++ /dev/null@@ -1,93 +0,0 @@-use std::{cell::RefCell, collections::HashMap};--use mpsgraph::TensorData;-use objc2::rc::Retained;--use super::{ArrayId, ForwardPassState};-use crate::backends::metal::array::MetalArray;--#[derive(Clone)]-pub struct IOArrays {- input_arrays: Box<[ArrayId]>,- output_arrays: Box<[ArrayId]>,-}--pub struct MPSGraphFeeds {- pub inputs: Box<[Retained<TensorData>]>,- pub outputs: Box<[Retained<TensorData>]>,-}--pub struct KernelFeeds {- pub inputs: Box<[RefCell<MetalArray>]>,- pub outputs: Box<[RefCell<MetalArray>]>,-}--impl IOArrays {- pub fn new(- input_arrays: Box<[ArrayId]>,- output_arrays: Box<[ArrayId]>,- ) -> Self {- Self {- input_arrays,- output_arrays,- }- }-- fn all_ids(&self) -> Box<[ArrayId]> {- let mut potentially_duplicated_results: Vec<ArrayId> = self- .input_arrays- .iter()- .copied()- .chain(self.output_arrays.iter().copied())- .collect();- potentially_duplicated_results.sort();- potentially_duplicated_results.dedup();- potentially_duplicated_results.into()- }-- pub unsafe fn get_mpsgraph_feeds(- &self,- state: &mut ForwardPassState,- ) -> MPSGraphFeeds {- unsafe {- let all_ids = self.all_ids();- let all_tensor_datas =- state.arrays(&all_ids).into_iter().map(|a_rc_refcell| {- a_rc_refcell.borrow_mut().to_mps_tensor_data()- });- let id_to_result =- HashMap::<ArrayId, Retained<TensorData>>::from_iter(- all_ids.into_iter().zip(all_tensor_datas),- );- let inputs: Box<[_]> = self- .input_arrays- .iter()- .map(|id| id_to_result[id].clone())- .collect();- let outputs: Box<[_]> = self- .output_arrays- .iter()- .map(|id| id_to_result[id].clone())- .collect();- MPSGraphFeeds {- inputs,- outputs,- }- }- }-- pub unsafe fn get_kernel_feeds(- &self,- state: &mut ForwardPassState,- ) -> KernelFeeds {- let inputs: Box<[RefCell<MetalArray>]> =- state.arrays(&self.input_arrays);- let outputs: Box<[RefCell<MetalArray>]> =- state.arrays(&self.output_arrays);-- KernelFeeds {- inputs,- outputs,- }- }-}diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rsindex 37b24c7d7..72714b73b 100644--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs@@ -1,9 +1,7 @@pub mod cache_layers;mod encoder_resolver;-mod io_arrays;pub mod kv_cache_layer;mod model_shape;-mod mpsgraph_block;mod scratch_buffers;pub mod short_conv_layer;pub mod ssm_layer;@@ -13,12 +11,10 @@ pub mod traces;pub use cache_layers::{CacheLayer, CacheLayers};pub use encoder_resolver::EncoderResolver;-pub use io_arrays::IOArrays;pub use kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,};pub use model_shape::ModelShape;-pub use mpsgraph_block::MPSGraphBlock;pub use scratch_buffers::ScratchBuffers;pub use short_conv_layer::ShortConvLayer;pub use ssm_layer::SSMLayer;diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rsdeleted file mode 100644index f285c3f6e..000000000--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs+++ /dev/null@@ -1,76 +0,0 @@-use mpsgraph::{- CommandBuffer as MPSCommandBuffer, Executable,- ExecutableExecutionDescriptor, TensorData,-};-use objc2::rc::Retained;--use super::{- EncodableBlock, EncodingParameters, ForwardPassState, io_arrays::IOArrays,-};--pub struct MPSGraphBlock {- executable: Retained<Executable>,- execution_descriptor: Retained<ExecutableExecutionDescriptor>,- arguments: IOArrays,-}--impl MPSGraphBlock {- pub fn new(- executable: Retained<Executable>,- execution_descriptor: Retained<ExecutableExecutionDescriptor>,- arguments: IOArrays,- ) -> Self {- Self {- executable,- execution_descriptor,- arguments,- }- }-}--impl EncodableBlock for MPSGraphBlock {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- unsafe {- let feeds = self.arguments.get_mpsgraph_feeds(state);-- let inputs_refs: Vec<&TensorData> =- feeds.inputs.iter().map(|retained_td| &**retained_td).collect();- let inputs_slice: &[&TensorData] = &inputs_refs;-- let outputs_refs: Vec<&TensorData> = feeds- .outputs- .iter()- .map(|retained_td| &**retained_td)- .collect();- let outputs_slice: &[&TensorData] = &outputs_refs;-- let maybe_outputs_slice = if outputs_slice.is_empty() {- None- } else {- Some(outputs_slice)- };-- let execution_descriptor = self.execution_descriptor.clone();- execution_descriptor- .set_enable_commit_and_continue(parameters.enable_commit);-- let root_command_buffer =- command_buffer.root_command_buffer().to_owned();- let _ = self.executable.encode_to_command_buffer(- command_buffer,- inputs_slice,- maybe_outputs_slice,- Some(&execution_descriptor),- );- if parameters.wait_until_completed {- command_buffer.commit_and_continue();- root_command_buffer.wait_until_completed();- }- }- }-}diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rsindex 86654b28f..aabb8a89f 100644--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs@@ -1020,7 +1020,7 @@ impl ForwardPassState {pub fn encode_copy_array(&self,- command_buffer: &mpsgraph::CommandBuffer,+ command_buffer: &metal::CommandBufferRef,source_array_id: ArrayId,destination_array: RefCell<MetalArray>,) {@@ -1037,9 +1037,7 @@ impl ForwardPassState {src_borrow.size_in_bytes());- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let blit_encoder = mtl_command_buffer.new_blit_command_encoder();+ let blit_encoder = command_buffer.new_blit_command_encoder();blit_encoder.copy_from_buffer(&src_buf,0,diff --git a/crates/uzu/src/backends/metal/graph/common.rs b/crates/uzu/src/backends/metal/graph/common.rsdeleted file mode 100644index 078c10c68..000000000--- a/crates/uzu/src/backends/metal/graph/common.rs+++ /dev/null@@ -1,257 +0,0 @@-use std::rc::Rc;--use mpsgraph::{Graph, ShapedType, Tensor};-use objc2::rc::Retained;-use thiserror::Error;--use crate::{- Array, DataType,- backends::metal::{MTLContext, MetalArray},- config::Activation,- parameters::{ParameterLoaderError, ParameterTree},-};--pub fn data_type_of(tensor: &Tensor) -> DataType {- DataType::from(tensor.data_type())-}--#[derive(Debug, Error)]-pub enum GraphConstructionError {- #[error("Failed to load weights: {0}")]- ParameterLoaderError(#[from] ParameterLoaderError),- #[error(- "Incompatible shapes. Node {node_name} at subtree {node_path:?} expected to have shape {expected:?} but got {actual:?}"- )]- IncompatibleShapes {- node_path: Option<String>,- node_name: String,- expected: Box<[usize]>,- actual: Box<[usize]>,- },- #[error(- "Incompatible data types. Node {node_name} at subtree {node_path:?} expected to have data type {expected:?} but got {actual:?}"- )]- IncompatibleDataTypes {- node_path: Option<String>,- node_name: String,- expected: DataType,- actual: DataType,- },- #[error("Failed to run callable")]- CallableResultError {- name: String,- },-}--pub fn activation(- graph: &Graph,- config: &Activation,- input: &Tensor,- data_type: DataType,-) -> Retained<Tensor> {- match config {- Activation::SiLU {- ..- } => {- let sigmoid_result = graph.sigmoid(input, Some("sigmoid"));- graph.multiplication(input, &sigmoid_result, Some("silu_output"))- },- Activation::Gelu => gelu(graph, input, data_type, true),- Activation::Identity => graph.addition(- input,- &graph.constant_with_scalar(0.0_f64, None, data_type.into()),- Some("identity_output"),- ),- }-}--pub fn gelu(- graph: &Graph,- input: &Tensor,- data_type: DataType,- approximate: bool,-) -> Retained<Tensor> {- if approximate {- let input_power_2 = graph.multiplication(input, input, None);- let input_power_3 = graph.multiplication(&input_power_2, input, None);- let input_power_3_multiply_const = graph.multiplication(- &graph.constant_with_scalar(- 0.044715 as f64,- None,- data_type.into(),- ),- &input_power_3,- None,- );- let input_add_input_power_3_multiply_const =- graph.addition(input, &input_power_3_multiply_const, None);- let tanh = graph.tanh(- &graph.multiplication(- &graph.constant_with_scalar(- (2.0 / std::f64::consts::PI).sqrt(),- None,- data_type.into(),- ),- &input_add_input_power_3_multiply_const,- None,- ),- None,- );- let tanh_add_one = graph.addition(- &graph.constant_with_scalar(1.0 as f64, None, data_type.into()),- &tanh,- None,- );- let input_multiply_tanh_add_one =- graph.multiplication(input, &tanh_add_one, None);- let result = graph.multiplication(- &input_multiply_tanh_add_one,- &graph.constant_with_scalar(0.5 as f64, None, data_type.into()),- None,- );- result- } else {- let input_negative = graph.multiplication(- &graph.constant_with_scalar(-1.0 as f64, None, data_type.into()),- input,- None,- );- let input_negative_divide_sqrt_2 = graph.division(- &input_negative,- &graph.constant_with_scalar(- (2.0 as f64).sqrt(),- None,- data_type.into(),- ),- None,- );- let erf = graph.erf(&input_negative_divide_sqrt_2, None);- let erf_negative = graph.multiplication(- &graph.constant_with_scalar(-1.0 as f64, None, data_type.into()),- &erf,- None,- );- let erfc = graph.addition(- &graph.constant_with_scalar(1.0 as f64, None, data_type.into()),- &erf_negative,- None,- );- let input_divide_2 = graph.division(- &input,- &graph.constant_with_scalar(2.0 as f64, None, data_type.into()),- None,- );- let result = graph.multiplication(&input_divide_2, &erfc, None);- result- }-}--pub fn placeholder(- graph: &Graph,- shape: &[isize],- data_type: DataType,-) -> Retained<Tensor> {- graph.placeholder(Some(shape), data_type.into(), Some("input"))-}--pub fn shaped_type(- shape: &[isize],- data_type: DataType,-) -> Retained<ShapedType> {- ShapedType::new_with_shape_data_type(Some(shape), data_type.into())-}--fn map_last_dimension<F: FnMut(usize) -> usize>(- mut f: F,- shape: &[usize],-) -> Box<[usize]> {- shape- .iter()- .enumerate()- .map(|(i, dim)| {- if i == shape.len() - 1 {- f(*dim)- } else {- *dim- }- })- .collect()-}--fn unpacked_shape_from_packed_shape(packed_shape: &[usize]) -> Box<[usize]> {- map_last_dimension(|dim| dim * 2, packed_shape)-}--fn packed_shape_from_unpacked_shape(packed_shape: &[usize]) -> Box<[usize]> {- map_last_dimension(|dim| dim / 2, packed_shape)-}--pub fn i4_constant_from_packed_u8_array(- graph: &Graph,- data: &MetalArray,-) -> Retained<Tensor> {- assert!(data.data_type() == DataType::U8);- graph.constant_with_data(- data.buffer(),- &unpacked_shape_from_packed_shape(data.shape()),- DataType::I4.into(),- )-}--pub fn load_constant(- graph: &Graph,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- name: &str,- expected_shape: &[usize],- expected_data_type: DataType,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let parameter = parameter_tree.leaf(name)?;- let result = if expected_data_type == DataType::U4 {- if parameter.data_type() != DataType::U8 {- return Err(GraphConstructionError::IncompatibleDataTypes {- node_path: parameter_tree.path_prefix().map(str::to_string),- node_name: name.to_string(),- expected: DataType::U8,- actual: parameter.data_type(),- });- };- let packed_expected_shape =- packed_shape_from_unpacked_shape(expected_shape);- if parameter.shape() != &*packed_expected_shape {- return Err(GraphConstructionError::IncompatibleShapes {- node_path: parameter_tree.path_prefix().map(str::to_string),- node_name: name.to_string(),- expected: packed_expected_shape.into(),- actual: parameter.shape().into(),- });- };- graph.constant_with_data(- parameter.buffer(),- expected_shape,- DataType::U4.into(),- )- } else {- if parameter.shape() != expected_shape {- return Err(GraphConstructionError::IncompatibleShapes {- node_path: parameter_tree.path_prefix().map(str::to_string),- node_name: name.to_string(),- expected: expected_shape.into(),- actual: parameter.shape().into(),- });- }- if parameter.data_type() != expected_data_type {- return Err(GraphConstructionError::IncompatibleDataTypes {- node_path: parameter_tree.path_prefix().map(str::to_string),- node_name: name.to_string(),- expected: expected_data_type,- actual: parameter.data_type(),- });- };- graph.constant_with_data(- parameter.buffer(),- parameter.shape(),- parameter.data_type().into(),- )- };- Ok(result)-}diff --git a/crates/uzu/src/backends/metal/graph/embedding.rs b/crates/uzu/src/backends/metal/graph/embedding.rsdeleted file mode 100644index e712a35dd..000000000--- a/crates/uzu/src/backends/metal/graph/embedding.rs+++ /dev/null@@ -1,49 +0,0 @@-use mpsgraph::{DequantizationArguments, Graph, Tensor};-use objc2::rc::Retained;--use super::GraphConstructionError;-use crate::config::EmbeddingConfig;--pub fn embeddings_dequantize_weights_subgraph(- graph: &Retained<Graph>,- weights: &Retained<Tensor>,- scales: &Retained<Tensor>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let result = graph.dequantize(- &weights,- DequantizationArguments::ScaleTensorZeroPointDataTypeAxis {- scale_tensor: &scales,- zero_point: 0.0,- data_type: scales.data_type(),- axis: 0,- },- None,- );- Ok(result)-}--pub fn embeddings_embed_subgraph(- graph: &Retained<Graph>,- config: &EmbeddingConfig,- input: &Retained<Tensor>,- weights: &Retained<Tensor>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let result = graph.gather_with_updates(&weights, input, 0, 0, None);- if let Some(scale) = config.common().input_scale {- let scale_tensor =- graph.constant_with_scalar(scale as f64, None, result.data_type());- let scaled_result = graph.multiplication(&result, &scale_tensor, None);- Ok(scaled_result)- } else {- Ok(result)- }-}--pub fn embeddings_readout_subgraph(- graph: &Retained<Graph>,- input: &Retained<Tensor>,- weights: &Retained<Tensor>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let result = graph.matrix_multiplication(input, weights, None);- Ok(result)-}diff --git a/crates/uzu/src/backends/metal/graph/linear.rs b/crates/uzu/src/backends/metal/graph/linear.rsdeleted file mode 100644index 1422d5b49..000000000--- a/crates/uzu/src/backends/metal/graph/linear.rs+++ /dev/null@@ -1,398 +0,0 @@-use std::rc::Rc;--use mpsgraph::{DequantizationArguments, Graph, Tensor};-use objc2::rc::Retained;--use super::{super::MTLContext, GraphConstructionError, load_constant};-use crate::{- DataType,- config::{ConfigDataType, LinearConfig, QuantizationConfig},- parameters::ParameterTree,-};--fn full_precision_weights_subgraph<const N: usize>(- graph: &Graph,- precision: DataType,- input_dim: usize,- output_dims: [usize; N],- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let output_dim_sum: usize = output_dims.iter().sum();- load_constant(- graph,- parameter_tree,- "weights",- &[output_dim_sum, input_dim],- precision,- )-}--fn quantized_weights_subgraph<const N: usize>(- graph: &Graph,- config: &QuantizationConfig,- input_dim: usize,- output_dims: [usize; N],- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let output_dim_sum: usize = output_dims.iter().sum();-- let weights = load_constant(- graph,- parameter_tree,- "weights",- &[input_dim, output_dim_sum],- config.weight_quantization_mode.into(),- )?;-- let scales = load_constant(- graph,- parameter_tree,- "scales",- &[input_dim, output_dim_sum / config.group_size],- config.activation_precision.into(),- )?;-- let zero_points = load_constant(- graph,- parameter_tree,- "zero_points",- &[input_dim, output_dim_sum / config.group_size],- config.weight_quantization_mode.into(),- )?;-- let result = graph.dequantize(- &weights,- DequantizationArguments::ScaleTensorZeroPointTensorDataType {- scale_tensor: &scales,- zero_point_tensor: &zero_points,- data_type: scales.data_type(),- },- None,- );- Ok(result)-}--fn mlx_quantized_weights_subgraph<const N: usize>(- graph: &Graph,- config: &QuantizationConfig,- input_dim: usize,- output_dims: [usize; N],- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let output_dim_sum: usize = output_dims.iter().sum();-- let weights = load_constant(- graph,- parameter_tree,- "weights",- &[input_dim, output_dim_sum],- config.weight_quantization_mode.into(),- )?;-- let scales = load_constant(- graph,- parameter_tree,- "scales",- &[input_dim, output_dim_sum / config.group_size],- config.activation_precision.into(),- )?;-- let deq_biases = load_constant(- graph,- parameter_tree,- "deq_biases",- &[input_dim, output_dim_sum / config.group_size],- config.activation_precision.into(),- )?;-- let result = graph.dequantize(- &weights,- DequantizationArguments::ScaleTensorZeroPointTensorDataType {- scale_tensor: &scales,- zero_point_tensor: &deq_biases,- data_type: scales.data_type(),- },- None,- );- Ok(result)-}--fn biases_subgraph<const N: usize>(- graph: &Graph,- precision: DataType,- output_dims: [usize; N],- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let output_dim_sum: usize = output_dims.iter().sum();- load_constant(graph, parameter_tree, "biases", &[output_dim_sum], precision)-}--struct LoraWeights {- down_weights: Retained<Tensor>,- up_weights: Box<[Retained<Tensor>]>,-}--fn lora_weights_subgraph<const N: usize>(- graph: &Graph,- precision: DataType,- input_dim: usize,- output_dims: [usize; N],- lora_rank: usize,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<LoraWeights, GraphConstructionError> {- let down_weights = load_constant(- graph,- parameter_tree,- "down_weights",- &[input_dim, lora_rank * N],- precision,- )?;- let up_weights = output_dims- .iter()- .enumerate()- .map(|(i, output_dim)| {- load_constant(- graph,- parameter_tree,- &format!("up_weights.{}", i),- &[lora_rank, *output_dim],- precision,- )- })- .collect::<Result<Box<[_]>, _>>()?;- Ok(LoraWeights {- down_weights,- up_weights: up_weights,- })-}--fn full_precision_matmul_subgraph<const N: usize>(- graph: &Graph,- precision: DataType,- input_dim: usize,- output_dims: [usize; N],- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let weights = full_precision_weights_subgraph::<N>(- graph,- precision,- input_dim,- output_dims,- parameter_tree,- )?;- Ok(graph.transpose(- &graph.matrix_multiplication(- &weights,- &graph.transpose(input, &[1, 0], None),- None,- ),- &[1, 0],- None,- ))-}--fn quantized_matmul_subgraph<const N: usize>(- graph: &Graph,- config: &QuantizationConfig,- input_dim: usize,- output_dims: [usize; N],- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let weights = quantized_weights_subgraph::<N>(- graph,- config,- input_dim,- output_dims,- parameter_tree,- )?;- Ok(graph.matrix_multiplication(input, &weights, None))-}--fn mlx_quantized_matmul_subgraph<const N: usize>(- graph: &Graph,- config: &QuantizationConfig,- input_dim: usize,- output_dims: [usize; N],- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let weights = mlx_quantized_weights_subgraph::<N>(- graph,- config,- input_dim,- output_dims,- parameter_tree,- )?;- Ok(graph.matrix_multiplication(input, &weights, None))-}--fn lora_subgraph<const N: usize>(- graph: &Graph,- precision: DataType,- lora_rank: usize,- input_dim: usize,- output_dims: [usize; N],- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let lora_weights = lora_weights_subgraph(- graph,- precision,- input_dim,- output_dims,- lora_rank,- parameter_tree,- )?;-- let lora_inner = graph.transpose(- &graph.matrix_multiplication(- &graph.transpose(&lora_weights.down_weights, &[1, 0], None),- &graph.transpose(input, &[1, 0], None),- None,- ),- &[1, 0],- None,- );-- let lora_inner_splits =- graph.split_num_splits(&lora_inner, output_dims.len() as u64, 1, None);- assert_eq!(lora_inner_splits.len(), lora_weights.up_weights.len());-- let lora_outputs = lora_inner_splits- .iter()- .zip(lora_weights.up_weights.iter())- .map(|(lora_inner_split, up_weights)| {- graph.transpose(- &graph.matrix_multiplication(- &graph.transpose(&up_weights, &[1, 0], None),- &graph.transpose(&lora_inner_split, &[1, 0], None),- None,- ),- &[1, 0],- None,- )- })- .collect::<Box<[Retained<Tensor>]>>();-- Ok(graph.concat_tensors(- &lora_outputs- .iter()- .map(|tensor| &**tensor)- .collect::<Box<[&Tensor]>>(),- 1,- false,- None,- ))-}--fn qlora_matmul_subgraph<const N: usize>(- graph: &Graph,- lora_rank: usize,- lora_scale: f32,- config: &QuantizationConfig,- input_dim: usize,- output_dims: [usize; N],- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let quantized_matmul_result = quantized_matmul_subgraph::<N>(- graph,- config,- input_dim,- output_dims,- input,- parameter_tree,- )?;- let lora_result = lora_subgraph::<N>(- graph,- config.activation_precision.into(),- lora_rank,- input_dim,- output_dims,- input,- parameter_tree,- )?;- let scaled_lora_result = graph.multiplication(- &lora_result,- &graph.constant_with_scalar(- lora_scale as f64,- None,- <ConfigDataType as Into<DataType>>::into(- config.activation_precision,- )- .into(),- ),- None,- );- Ok(graph.addition(&quantized_matmul_result, &scaled_lora_result, None))-}--pub fn linear_subgraph<const N: usize>(- graph: &Graph,- config: &LinearConfig,- input_dim: usize,- output_dims: [usize; N],- has_biases: bool,- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let matmul_result = match config {- LinearConfig::FullPrecision {- precision,- } => full_precision_matmul_subgraph(- graph,- (*precision).into(),- input_dim,- output_dims,- input,- parameter_tree,- )?,- LinearConfig::Quantized(quantization_config) => {- quantized_matmul_subgraph(- graph,- quantization_config,- input_dim,- output_dims,- input,- parameter_tree,- )?- },- LinearConfig::MLXQuantized(quantization_config) => {- mlx_quantized_matmul_subgraph(- graph,- quantization_config,- input_dim,- output_dims,- input,- parameter_tree,- )?- },- LinearConfig::QLoRA {- quantization: quantization_config,- lora_rank,- lora_scale,- } => qlora_matmul_subgraph(- graph,- *lora_rank,- *lora_scale,- quantization_config,- input_dim,- output_dims,- input,- parameter_tree,- )?,- };- if has_biases {- let biases = biases_subgraph(- graph,- config.activation_precision().into(),- output_dims,- parameter_tree,- )?;- Ok(graph.addition(&matmul_result, &biases, None))- } else {- Ok(matmul_result)- }-}diff --git a/crates/uzu/src/backends/metal/graph/mlp.rs b/crates/uzu/src/backends/metal/graph/mlp.rsdeleted file mode 100644index 509e30dc3..000000000--- a/crates/uzu/src/backends/metal/graph/mlp.rs+++ /dev/null@@ -1,64 +0,0 @@-use std::rc::Rc;--use mpsgraph::{Graph, Tensor};-use objc2::rc::Retained;--use super::{- super::MTLContext, GraphConstructionError, common::activation,- linear::linear_subgraph,-};-use crate::{config::MLPConfig, parameters::ParameterTree};--pub fn mlp_subgraph(- graph: &Graph,- config: &MLPConfig,- model_dim: usize,- hidden_dim: usize,- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- match config {- MLPConfig::Dense(dense) => {- let up_tree = parameter_tree.subtree("up_projection")?;- let down_tree = parameter_tree.subtree("down_projection")?;-- let fused_hidden_gate = linear_subgraph(- graph,- &dense.linear_config,- model_dim,- [hidden_dim, hidden_dim],- false,- input,- &up_tree,- )?;-- let split_results =- graph.split_num_splits(&fused_hidden_gate, 2, 1, None);- let up_proj = &split_results[0];- let gate = &split_results[1];-- let activated_gate = activation(- graph,- &dense.activation,- &*gate,- dense.linear_config.activation_precision().into(),- );- let hidden = graph.multiplication(&activated_gate, up_proj, None);-- let result = linear_subgraph(- graph,- &dense.linear_config,- hidden_dim,- [model_dim],- false,- &*hidden,- &down_tree,- )?;-- Ok(result)- },- MLPConfig::MixtureOfExperts(_moe) => {- unreachable!("MoE uses MoeBlockEncodable, not MPSGraph")- },- }-}diff --git a/crates/uzu/src/backends/metal/graph/mod.rs b/crates/uzu/src/backends/metal/graph/mod.rsdeleted file mode 100644index ddb1f870f..000000000--- a/crates/uzu/src/backends/metal/graph/mod.rs+++ /dev/null@@ -1,12 +0,0 @@-pub mod common;-use common::{GraphConstructionError, load_constant};-pub use common::{placeholder, shaped_type};-mod linear;-pub use linear::linear_subgraph;-mod mlp;-pub use mlp::mlp_subgraph;-mod embedding;-pub use embedding::{- embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,- embeddings_readout_subgraph,-};diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metalnew file mode 100644index 000000000..fb5e88d00--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal@@ -0,0 +1,71 @@+#include <metal_stdlib>+using namespace metal;++enum ActivationType : ushort {+ ACT_SILU = 0,+ ACT_GELU = 1,+};++template <typename T>+inline T silu(T x) {+ float xf = float(x);+ float yf = xf / (1.0f + exp(-xf));+ return T(yf);+}++template <typename T>+inline T gelu_approx(T x) {+ constexpr float k0 = 0.044715f;+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)+ float xf = float(x);+ float yf = 0.5f * xf * (1.0f + tanh(k1 * (xf + k0 * xf * xf * xf)));+ return T(yf);+}++template <typename T>+inline T activate(T x, ushort act) {+ if (act == ACT_SILU)+ return silu(x);+ return gelu_approx(x);+}++template <typename T>+[[kernel]] void activation(+ const device T* input [[buffer(0)]],+ device T* output [[buffer(1)]],+ const constant int& N [[buffer(2)]],+ const constant ushort& act_type [[buffer(3)]],+ uint tid [[thread_position_in_grid]]+) {+ if (int(tid) >= N)+ return;+ output[tid] = activate(input[tid], act_type);+}++// Explicit instantiations+template [[host_name("activation_f16")]] [[kernel]] void+activation<half>(+ const device half* input [[buffer(0)]],+ device half* output [[buffer(1)]],+ const constant int& N [[buffer(2)]],+ const constant ushort& act_type [[buffer(3)]],+ uint tid [[thread_position_in_grid]]+);++template [[host_name("activation_f32")]] [[kernel]] void+activation<float>(+ const device float* input [[buffer(0)]],+ device float* output [[buffer(1)]],+ const constant int& N [[buffer(2)]],+ const constant ushort& act_type [[buffer(3)]],+ uint tid [[thread_position_in_grid]]+);++template [[host_name("activation_bf16")]] [[kernel]] void+activation<bfloat>(+ const device bfloat* input [[buffer(0)]],+ device bfloat* output [[buffer(1)]],+ const constant int& N [[buffer(2)]],+ const constant ushort& act_type [[buffer(3)]],+ uint tid [[thread_position_in_grid]]+);diff --git a/crates/uzu/src/backends/metal/kernel/activation/mod.rs b/crates/uzu/src/backends/metal/kernel/activation/mod.rsnew file mode 100644index 000000000..c191d30d6--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/activation/mod.rs@@ -0,0 +1,90 @@+use std::mem::size_of;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+ config::Activation,+};++pub struct ActivationKernel {+ pipeline: MTLComputePipelineState,+}++impl ActivationKernel {+ fn kernel_name_for_type(+ data_type: DataType+ ) -> Result<&'static str, MTLError> {+ match data_type {+ DataType::F16 => Ok("activation_f16"),+ DataType::F32 => Ok("activation_f32"),+ DataType::BF16 => Ok("activation_bf16"),+ other => Err(MTLError::Generic(format!(+ "Unsupported dtype for activation: {:?}",+ other+ ))),+ }+ }++ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ ) -> Result<Self, MTLError> {+ let fn_name = Self::kernel_name_for_type(data_type)?;+ let (pipeline, _) =+ context.compute_pipeline_state_with_reflection(fn_name, None)?;+ Ok(Self {+ pipeline,+ })+ }++ fn act_code(act: &Activation) -> u16 {+ match act {+ Activation::SiLU {+ ..+ } => 0,+ Activation::Gelu => 1,+ Activation::Identity => {+ panic!("Identity activation is not supported for kernel")+ },+ }+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ activation: &Activation,+ input_buffer: &MTLBuffer,+ output_buffer: &MTLBuffer,+ n: usize,+ ) -> Result<(), MTLError> {+ let act_code = Self::act_code(activation);+ let threads_per_tg = 256u64;+ let threadgroups = (n as u64 + threads_per_tg - 1) / threads_per_tg;++ encoder.set_compute_pipeline_state(&self.pipeline);+ encoder.set_buffer(0, Some(input_buffer), 0);+ encoder.set_buffer(1, Some(output_buffer), 0);+ encoder.set_bytes(+ 2,+ size_of::<i32>() as u64,+ &(n as i32) as *const i32 as *const _,+ );+ encoder.set_bytes(+ 3,+ size_of::<u16>() as u64,+ &act_code as *const u16 as *const _,+ );++ encoder.dispatch_thread_groups(+ MTLSize::new(threadgroups, 1, 1),+ MTLSize::new(threads_per_tg, 1, 1),+ );+ Ok(())+ }+}+diff --git a/crates/uzu/src/backends/metal/kernel/data_type.rs b/crates/uzu/src/backends/metal/kernel/data_type.rsindex 1e28cf2e8..1237a4185 100644--- a/crates/uzu/src/backends/metal/kernel/data_type.rs+++ b/crates/uzu/src/backends/metal/kernel/data_type.rs@@ -17,9 +17,9 @@ impl KernelDataType {}}-impl Into<DataType> for KernelDataType {- fn into(self) -> DataType {- match self {+impl From<KernelDataType> for DataType {+ fn from(val: KernelDataType) -> Self {+ match val {KernelDataType::BFloat16 => DataType::BF16,KernelDataType::Float16 => DataType::F16,KernelDataType::Float32 => DataType::F32,diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rsindex 647eff7f4..16f9c1ae9 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs@@ -7,7 +7,7 @@ use metal::{use super::super::MTLContext;use crate::{- DataType, backends::metal::encodable_block::QuantizedEmbeddingError,+ DataType, backends::metal::encodable_block::EmbeddingError,config::QuantizationMode,};@@ -32,13 +32,13 @@ impl FullPrecisionEmbeddingLookupKernel {pub fn new(mtl_context: &MTLContext,data_type: DataType,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {let dtype_suffix = match data_type {DataType::F32 => "f32",DataType::F16 => "f16",DataType::BF16 => "bf16",other => {- return Err(QuantizedEmbeddingError::UnsupportedDataType(+ return Err(EmbeddingError::UnsupportedDataType(other,));},@@ -48,7 +48,7 @@ impl FullPrecisionEmbeddingLookupKernel {let (pipeline, _) = mtl_context.compute_pipeline_state_with_reflection(&kernel_name, None)- .map_err(QuantizedEmbeddingError::MetalError)?;+ .map_err(EmbeddingError::MetalError)?;Ok(Self {pipeline,@@ -59,7 +59,7 @@ impl FullPrecisionEmbeddingLookupKernel {&self,encoder: &ComputeCommandEncoderRef,args: FullPrecisionEmbeddingLookupArguments,- ) -> Result<(), QuantizedEmbeddingError> {+ ) -> Result<(), EmbeddingError> {encoder.set_compute_pipeline_state(&self.pipeline);encoder.set_buffer(0, Some(args.token_ids_buffer), 0);@@ -126,13 +126,13 @@ impl QuantizedEmbeddingLookupKernel {mtl_context: &MTLContext,data_type: DataType,mode: QuantizationMode,- ) -> Result<Self, QuantizedEmbeddingError> {+ ) -> Result<Self, EmbeddingError> {let dtype_suffix = match data_type {DataType::F32 => "f32",DataType::F16 => "f16",DataType::BF16 => "bf16",other => {- return Err(QuantizedEmbeddingError::UnsupportedDataType(+ return Err(EmbeddingError::UnsupportedDataType(other,));},@@ -149,7 +149,7 @@ impl QuantizedEmbeddingLookupKernel {let (pipeline, _) = mtl_context.compute_pipeline_state_with_reflection(&kernel_name, None)- .map_err(QuantizedEmbeddingError::MetalError)?;+ .map_err(EmbeddingError::MetalError)?;Ok(Self {pipeline,@@ -160,7 +160,7 @@ impl QuantizedEmbeddingLookupKernel {&self,encoder: &ComputeCommandEncoderRef,args: QuantizedEmbeddingLookupArguments,- ) -> Result<(), QuantizedEmbeddingError> {+ ) -> Result<(), EmbeddingError> {encoder.set_compute_pipeline_state(&self.pipeline);// Set buffersdiff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rsindex bae2543ef..5426957ab 100644--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs@@ -1,11 +1,10 @@use std::mem;use metal::{- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,+ Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer, CommandBufferRef,ComputeCommandEncoder, ComputePipelineState, MTLSize,foreign_types::{ForeignType, ForeignTypeRef},};-use mpsgraph::CommandBuffer as MPSCommandBuffer;use crate::backends::metal::{MTLContext,@@ -86,7 +85,7 @@ impl EncodableBlock for ScalePadNormalizeImage {fn encode(&self,_state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,+ command_buffer: &CommandBufferRef,_parameters: &EncodingParameters,) {let _ = command_buffer;diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 0528c0967..e21378a11 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,10 +1,11 @@+pub mod activation;pub mod attention;mod data_type;pub mod embedding;pub mod kv_cache_update;pub mod layer_norm;-pub mod matmul;pub mod mask_update;+pub mod matmul;pub mod media_kernels;pub mod mlp;pub mod moe;@@ -21,6 +22,7 @@ mod tensor_add_swap;mod tensor_copy;pub mod token_copy;+pub use activation::ActivationKernel;pub use attention::{AttentionError, AttentionKernel, AttentionKernelVariant,AttentionSinglePassArguments, AttentionTwoPassArguments,@@ -29,8 +31,8 @@ pub use attention::{pub use data_type::KernelDataType;pub use kv_cache_update::KVCacheUpdate;pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};-pub use matmul::{MatmulArguments, MatmulKernel};pub use mask_update::MaskUpdateKernel;+pub use matmul::{MatmulArguments, MatmulKernel};pub use moe::{MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,MoeCountsOffsetsFusedError, MoeCountsOffsetsFusedKernel,diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rsindex fd6b0afd4..f42a4da4d 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs@@ -1,8 +1,8 @@use std::mem::size_of;use metal::{- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,+ Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState,};use super::{@@ -37,7 +37,7 @@ impl TensorAddBias {output: &MTLBuffer,num_cols: usize,total_len: usize,- command_buffer: &MTLCommandBuffer,+ command_buffer: &CommandBufferRef,predicate: Option<&MTLBuffer>,) {let encoder = command_buffer.new_compute_command_encoder();diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rsindex 1dae0f2c3..d51749746 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs@@ -1,7 +1,7 @@use std::mem::size_of;use metal::{- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,+ Buffer as MTLBuffer, CommandBufferRef,ComputePipelineState as MTLComputePipelineState,};@@ -36,7 +36,7 @@ impl TensorAddSwapKernel {skip_buffer: &MTLBuffer,main_buffer: &MTLBuffer,length: usize,- command_buffer: &MTLCommandBuffer,+ command_buffer: &CommandBufferRef,) {let compute_encoder = command_buffer.new_compute_command_encoder();diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rsindex 743ff1d23..7d66ea8a3 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs@@ -1,7 +1,7 @@use std::mem::size_of;use metal::{- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,+ Buffer as MTLBuffer, CommandBufferRef,ComputePipelineState as MTLComputePipelineState,};@@ -35,7 +35,7 @@ impl TensorCopyKernel {source_buffer: &MTLBuffer,destination_buffer: &MTLBuffer,length: usize,- command_buffer: &MTLCommandBuffer,+ command_buffer: &CommandBufferRef,) {let compute_encoder = command_buffer.new_compute_command_encoder();compute_encoder.set_label("Tensor Copy");diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rsindex 3b85afdc4..74d5feee3 100644--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs@@ -6,9 +6,7 @@ use std::{rc::Rc,};-use metal::Event as MTLEvent;-use mpsgraph::CommandBuffer as MPSCommandBuffer;-use objc2::rc::Retained;+use metal::{CommandBuffer, Event as MTLEvent};use super::{CacheLayers, Decoder, KVCacheUpdate, KernelDataType, MTLContext,@@ -131,7 +129,7 @@ impl AsyncBuffers {pub struct LanguageModelGeneratorContext {pub mtl_context: Rc<MTLContext>,- pub command_buffer: Retained<MPSCommandBuffer>,+ pub command_buffer: CommandBuffer,pub cache_layers: Rc<RefCell<CacheLayers>>,pub shared_buffers: Rc<RefCell<SharedBuffers>>,@@ -163,8 +161,7 @@ impl LanguageModelGeneratorContext {let mtl_command_queue =mtl_device.new_command_queue_with_max_command_buffer_count(1024);- let command_buffer =- MPSCommandBuffer::from_command_queue(&mtl_command_queue);+ let command_buffer = mtl_command_queue.new_command_buffer().to_owned();let config_path = model_path.join("config.json");if !config_path.exists() {@@ -308,10 +305,7 @@ impl LanguageModelGeneratorContext {}pub fn reset_command_buffer(&mut self) {- objc2::rc::autoreleasepool(|_pool| {- self.command_buffer = MPSCommandBuffer::from_command_queue(- &self.mtl_context.command_queue,- );- });+ self.command_buffer =+ self.mtl_context.command_queue.new_command_buffer().to_owned();}}diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex 908ddbb34..c55780fed 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -6,7 +6,6 @@ mod context;pub mod encodable_block;pub mod error;pub mod forward_pass;-pub mod graph;pub mod image;pub mod kernel;mod language_model_generator_context;diff --git a/crates/uzu/src/config/common.rs b/crates/uzu/src/config/common.rsindex f99608cae..92bf13805 100644--- a/crates/uzu/src/config/common.rs+++ b/crates/uzu/src/config/common.rs@@ -13,9 +13,9 @@ pub enum ConfigDataType {Int8,}-impl Into<DataType> for ConfigDataType {- fn into(self) -> DataType {- match self {+impl From<ConfigDataType> for DataType {+ fn from(val: ConfigDataType) -> Self {+ match val {ConfigDataType::BFloat16 => DataType::BF16,ConfigDataType::Float16 => DataType::F16,ConfigDataType::Float32 => DataType::F32,@@ -64,9 +64,9 @@ impl QuantizationMode {}}-impl Into<DataType> for QuantizationMode {- fn into(self) -> DataType {- match self {+impl From<QuantizationMode> for DataType {+ fn from(val: QuantizationMode) -> Self {+ match val {QuantizationMode::UInt4 => DataType::U4,QuantizationMode::Int8 => DataType::I8,QuantizationMode::UInt8 => DataType::U8,diff --git a/crates/uzu/src/device/data_type.rs b/crates/uzu/src/device/data_type.rsindex 15ef1970f..d0ba5eb78 100644--- a/crates/uzu/src/device/data_type.rs+++ b/crates/uzu/src/device/data_type.rs@@ -1,6 +1,5 @@use bytemuck::Pod;use half::{bf16, f16};-use mpsgraph::DataType as MPSDataType;use num_traits::NumCast;#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Copy, Clone)]@@ -49,48 +48,6 @@ impl DataType {}}-impl From<MPSDataType> for DataType {- fn from(dtype: MPSDataType) -> Self {- match dtype {- MPSDataType::BFloat16 => DataType::BF16,- MPSDataType::Float16 => DataType::F16,- MPSDataType::Float32 => DataType::F32,- MPSDataType::Int4 => DataType::I4,- MPSDataType::UInt4 => DataType::U4,- MPSDataType::Int8 => DataType::I8,- MPSDataType::UInt8 => DataType::U8,- MPSDataType::Int16 => DataType::I16,- MPSDataType::UInt16 => DataType::U16,- MPSDataType::Int32 => DataType::I32,- MPSDataType::UInt32 => DataType::U32,- MPSDataType::Int64 => DataType::I64,- MPSDataType::UInt64 => DataType::U64,- _ => panic!("Unsupported MPS data type: {0:?}", dtype),- }- }-}--impl From<DataType> for MPSDataType {- fn from(dtype: DataType) -> Self {- match dtype {- DataType::BF16 => MPSDataType::BFloat16,- DataType::F16 => MPSDataType::Float16,- DataType::F32 => MPSDataType::Float32,- DataType::I4 => MPSDataType::Int4,- DataType::U4 => MPSDataType::UInt4,- DataType::I8 => MPSDataType::Int8,- DataType::U8 => MPSDataType::UInt8,- DataType::I16 => MPSDataType::Int16,- DataType::U16 => MPSDataType::UInt16,- DataType::I32 => MPSDataType::Int32,- DataType::U32 => MPSDataType::UInt32,- DataType::I64 => MPSDataType::Int64,- DataType::U64 => MPSDataType::UInt64,- _ => panic!("Unsupported MPS data type: {0:?}", dtype),- }- }-}-pub trait ArrayElement: NumCast + Pod {fn data_type() -> DataType;}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 8119aaf15..d58c95617 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -4,7 +4,6 @@ use std::{use block::ConcreteBlock;use itertools::{Either, Itertools, izip};-use mpsgraph::CommandBuffer;use super::{LanguageModelGeneratorContext,@@ -502,8 +501,7 @@ impl LanguageModelGenerator {}self.context.reset_command_buffer();- let root_command_buffer =- self.context.command_buffer.root_command_buffer();+ let root_command_buffer = &self.context.command_buffer;// Wait on previous pass if this is a continuationif is_continuation {@@ -611,7 +609,7 @@ impl LanguageModelGenerator {let block = block.copy();root_command_buffer.add_completed_handler(&block);- self.context.command_buffer.commit_and_continue();+ self.context.command_buffer.commit();Ok(())}@@ -704,8 +702,7 @@ impl LanguageModelGenerator {);}- let root_command_buffer =- self.context.command_buffer.root_command_buffer().to_owned();+ let root_command_buffer = self.context.command_buffer.clone();if !warmup {if !task.is_prefilling {@@ -717,9 +714,11 @@ impl LanguageModelGenerator {}}- self.context.command_buffer.commit_and_continue();+ self.context.command_buffer.commit();if allow_pre_encode {+ self.context.reset_command_buffer();+let next_task_key: String =task.encoded_task_key(self.tokens.len() + 1);@@ -773,11 +772,13 @@ impl LanguageModelGenerator {suffix_start: Option<usize>,wait_until_completed: bool,) {- let command_buffer = CommandBuffer::from_command_queue(- &self.context.mtl_context.command_queue,- );- let root_command_buffer =- command_buffer.root_command_buffer().to_owned();+ let command_buffer = self+ .context+ .mtl_context+ .command_queue+ .new_command_buffer()+ .to_owned();+ let root_command_buffer = command_buffer.to_owned();{let mut cache_layers = self.context.cache_layers.borrow_mut();@@ -792,7 +793,7 @@ impl LanguageModelGenerator {command_buffer.commit();if wait_until_completed {- command_buffer.root_command_buffer().wait_until_completed();+ command_buffer.wait_until_completed();}}diff --git a/crates/uzu/src/parameters/safetensors_metadata.rs b/crates/uzu/src/parameters/safetensors_metadata.rsindex f7628c223..e5f520b4a 100644--- a/crates/uzu/src/parameters/safetensors_metadata.rs+++ b/crates/uzu/src/parameters/safetensors_metadata.rs@@ -80,9 +80,9 @@ pub enum Dtype {U64,}-impl Into<DataType> for Dtype {- fn into(self) -> DataType {- match self {+impl From<Dtype> for DataType {+ fn from(val: Dtype) -> Self {+ match val {Dtype::F16 => DataType::F16,Dtype::BF16 => DataType::BF16,Dtype::F32 => DataType::F32,@@ -94,7 +94,7 @@ impl Into<DataType> for Dtype {Dtype::U64 => DataType::U64,Dtype::BOOL => DataType::I8,// Add mappings for other types as needed- _ => panic!("Unsupported dtype: {:?}", self),+ _ => panic!("Unsupported dtype: {:?}", val),}}}diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex d1c122344..7277373cd 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -1,7 +1,6 @@#![cfg(any(target_os = "macos", target_os = "ios"))]-use metal::Device;-use mpsgraph::CommandBuffer;+use metal::{CommandBuffer, Device};use uzu::{Array, DataType, DeviceContext,backends::metal::{@@ -223,10 +222,9 @@ fn run_scenario(},};- let command_buffer =- CommandBuffer::from_command_queue(&context.command_queue);+ let command_buffer = context.command_queue.new_command_buffer().to_owned();- let root_command_buffer = command_buffer.root_command_buffer().to_owned();+ let root_command_buffer = command_buffer.clone();layer.update_after_acceptance(&scenario.accepted_suffix_indices,scenario.suffix_start,diff --git a/crates/uzu/tests/split_tensor_datas_test.rs b/crates/uzu/tests/split_tensor_datas_test.rsdeleted file mode 100644index 2b801da02..000000000--- a/crates/uzu/tests/split_tensor_datas_test.rs+++ /dev/null@@ -1,282 +0,0 @@-use std::collections::HashMap;--use metal::{Device, MTLResourceOptions};-use mpsgraph::{- CommandBuffer, CompilationDescriptor, DataType as MPSDataType,- Device as MPSDevice, ExecutableExecutionDescriptor, Graph, ShapedType,- TensorData,-};-use ndarray::{Array2, ArrayView2};-use uzu::{Array, DataType, backends::metal::MetalArray};--fn sequential_f32(len: usize) -> Vec<f32> {- (0..len).map(|v| v as f32).collect()-}--fn ndarray_matmul(- m: usize,- k: usize,- n: usize,- a: &[f32],- b: &[f32],-) -> Vec<f32> {- assert_eq!(a.len(), m * k);- assert_eq!(b.len(), k * n);- let a_view = ArrayView2::from_shape((m, k), a).expect("Invalid A shape");- let b_view = ArrayView2::from_shape((k, n), b).expect("Invalid B shape");- let c: Array2<f32> = a_view.dot(&b_view);- let (vec, offset) = c.into_raw_vec_and_offset();- assert_eq!(offset, Some(0));- vec-}--#[test]-fn test_row_split_no_copy_simple() {- let device = match Device::system_default() {- Some(d) => d,- None => {- eprintln!("No Metal device; skipping test");- return;- },- };-- let shape = [4usize, 4usize];- let num_elems = shape[0] * shape[1];- let data = sequential_f32(num_elems);-- let buffer_size_bytes = (num_elems * std::mem::size_of::<f32>()) as u64;- let buffer = device.new_buffer_with_data(- data.as_ptr() as *const _,- buffer_size_bytes,- MTLResourceOptions::StorageModeShared,- );-- let mut array = unsafe { MetalArray::new(buffer, &shape, DataType::F32) };-- println!("About to call row_split_no_copy");- let splits = unsafe { array.row_split_no_copy(&[2, 2]) };- println!("Got {} splits", splits.len());- assert_eq!(splits.len(), 2);-}--#[test]-fn test_row_split_no_copy_shapes() {- let device = match Device::system_default() {- Some(d) => d,- None => {- eprintln!("No Metal device; skipping test");- return;- },- };-- let shape = [10usize, 6usize];- let num_elems = shape[0] * shape[1];- let data = sequential_f32(num_elems);-- let buffer_size_bytes = (num_elems * std::mem::size_of::<f32>()) as u64;- let buffer = device.new_buffer_with_data(- data.as_ptr() as *const _,- buffer_size_bytes,- MTLResourceOptions::StorageModeShared,- );-- let mut array = unsafe { MetalArray::new(buffer, &shape, DataType::F32) };-- let splits = unsafe { array.row_split_no_copy(&[2, 5, 3]) };- assert_eq!(splits.len(), 3);-- let expected_shapes =- vec![[2usize, 6usize], [5usize, 6usize], [3usize, 6usize]];- for (i, (split_array, expected_shape)) in- splits.iter().zip(expected_shapes.iter()).enumerate()- {- let shape = split_array.shape();- println!(- "Split {} has shape {:?}, expected {:?}",- i, shape, expected_shape- );- assert_eq!(shape, *expected_shape, "Shape mismatch for split {}", i);- }-- println!("All row splits have correct shapes!");-}--#[test]-fn test_row_split_no_copy_matmul() {- let device = match Device::system_default() {- Some(d) => d,- None => {- eprintln!("No Metal device; skipping test");- return;- },- };-- let a_shape = [5usize, 7usize];- let b_shape = [7usize, 4usize];-- let a_len = a_shape[0] * a_shape[1];- let b_len = b_shape[0] * b_shape[1];-- let a_data = sequential_f32(a_len);- let b_data: Vec<f32> = (0..b_len).map(|v| v as f32 * 0.5 + 1.0).collect();-- let b_buffer_size_bytes = (b_len * std::mem::size_of::<f32>()) as u64;- let b_buffer = device.new_buffer_with_data(- b_data.as_ptr() as *const _,- b_buffer_size_bytes,- MTLResourceOptions::StorageModeShared,- );- let b_td = TensorData::new_with_mtl_buffer(- &b_buffer,- &b_shape,- MPSDataType::Float32,- None,- );-- let baseline =- ndarray_matmul(a_shape[0], a_shape[1], b_shape[1], &a_data, &b_data);-- let a_buffer_size_bytes = (a_len * std::mem::size_of::<f32>()) as u64;- let a_buffer = device.new_buffer_with_data(- a_data.as_ptr() as *const _,- a_buffer_size_bytes,- MTLResourceOptions::StorageModeShared,- );- let mut a_array =- unsafe { MetalArray::new(a_buffer, &a_shape, DataType::F32) };-- let split_lengths = [2usize, 3usize];- let mut splits = unsafe { a_array.row_split_no_copy(&split_lengths) };- assert_eq!(splits.len(), split_lengths.len());-- let mut combined_c = Vec::with_capacity(baseline.len());- let mut offset = 0usize;- for (split_array, &rows) in splits.iter_mut().zip(split_lengths.iter()) {- let slice_start = offset * a_shape[1];- let slice_end = slice_start + rows * a_shape[1];- let slice_cpu = &a_data[slice_start..slice_end];- let expected =- ndarray_matmul(rows, a_shape[1], b_shape[1], slice_cpu, &b_data);-- let graph = Graph::new();-- let shape_a_isize = [rows as isize, a_shape[1] as isize];- let a_ph = graph.placeholder(- Some(&shape_a_isize),- MPSDataType::Float32,- Some("A"),- );-- let shape_b_isize = [a_shape[1] as isize, b_shape[1] as isize];- let b_ph = graph.placeholder(- Some(&shape_b_isize),- MPSDataType::Float32,- Some("B"),- );-- let c_tensor =- graph.matrix_multiplication(&a_ph, &b_ph, Some("MatMul"));-- let mps_device = MPSDevice::with_device(&device);- let compilation_descriptor = CompilationDescriptor::new();-- let a_shaped_type = ShapedType::new_with_shape_data_type(- Some(&shape_a_isize),- MPSDataType::Float32,- );- let b_shaped_type = ShapedType::new_with_shape_data_type(- Some(&shape_b_isize),- MPSDataType::Float32,- );-- let mut compile_feeds = HashMap::new();- compile_feeds.insert(a_ph.as_ref(), a_shaped_type.as_ref());- compile_feeds.insert(b_ph.as_ref(), b_shaped_type.as_ref());-- let targets = [c_tensor.as_ref()];- let executable = graph.compile(- &mps_device,- &compile_feeds,- &targets,- None,- Some(&compilation_descriptor),- );-- let result_size = rows * b_shape[1];- let result_buffer_size =- (result_size * std::mem::size_of::<f32>()) as u64;- let result_buffer = device.new_buffer(- result_buffer_size,- MTLResourceOptions::StorageModeShared,- );- let result_shape_usize = [rows, b_shape[1]];- let result_td = TensorData::new_with_mtl_buffer(- &result_buffer,- &result_shape_usize,- MPSDataType::Float32,- None,- );-- let split_td = unsafe { split_array.to_mps_tensor_data() };-- let inputs = [split_td.as_ref(), b_td.as_ref()];- let outputs = [result_td.as_ref()];-- let command_queue = device.new_command_queue();- let command_buffer = CommandBuffer::from_command_buffer(- &command_queue.new_command_buffer().to_owned(),- );-- let exec_desc = ExecutableExecutionDescriptor::new();- exec_desc.set_wait_until_completed(true);-- let _ = executable.encode_to_command_buffer(- &command_buffer,- &inputs,- Some(&outputs),- Some(&exec_desc),- );-- command_buffer.commit();- command_buffer.command_buffer().wait_until_completed();-- let gpu_result = unsafe {- let ptr = result_buffer.contents() as *const f32;- std::slice::from_raw_parts(ptr, result_size).to_vec()- };-- for (i, (&exp, &got)) in- expected.iter().zip(gpu_result.iter()).enumerate()- {- assert!(- (exp - got).abs() < 1e-4,- "Mismatch rows slice {} idx {}: expected {}, got {}",- rows,- i,- exp,- got- );- }-- combined_c.extend_from_slice(&expected);-- offset += rows;- }-- assert_eq!(combined_c.len(), baseline.len());- for (i, (expected, got)) in- baseline.iter().zip(combined_c.iter()).enumerate()- {- assert!(- (expected - got).abs() < 1e-6,- "Mismatch at index {}: expected {}, got {}",- i,- expected,- got- );- }-- println!(- "Matrix multiplication results match for split and unsplit tensors!"- );-}