Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove mpsgraph

eugene8aa9ea685e36
diff --git a/Cargo.lock b/Cargo.lock
index c3df3983a..b5369e596 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2191,19 +2191,6 @@ dependencies = [
"cxx",
]
-[[package]]
-name = "mpsgraph"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b3d4f4ee48894ff2d83808e8a0333f00eb7576375a5a77e8132f10ebb2a1f86"
-dependencies = [
- "bitflags 2.9.4",
- "block2",
- "metal",
- "objc2",
- "objc2-foundation",
-]
-
[[package]]
name = "multer"
version = "3.1.0"
@@ -4103,7 +4090,6 @@ dependencies = [
"metal",
"minijinja",
"minijinja-contrib",
- "mpsgraph",
"ndarray",
"num-traits",
"objc2",
diff --git a/crates/uzu/src/backends/metal/array.rs b/crates/uzu/src/backends/metal/array.rs
index 5ecbcea0d..555a70906 100644
--- a/crates/uzu/src/backends/metal/array.rs
+++ b/crates/uzu/src/backends/metal/array.rs
@@ -2,8 +2,6 @@ use std::ops::Range;
use half::{bf16, f16};
use metal::{Buffer as MTLBuffer, MTLResourceOptions};
-use mpsgraph::TensorData;
-use objc2::rc::Retained;
use crate::{Array, ArrayElement, DataType, array::array_size_in_bytes};
@@ -80,15 +78,6 @@ impl MetalArray {
self.offset
}
- pub unsafe fn to_mps_tensor_data(&mut self) -> Retained<TensorData> {
- TensorData::new_with_mtl_buffer(
- &self.buffer,
- &self.shape,
- self.data_type.into(),
- None,
- )
- }
-
pub unsafe fn mtl_buffer(&mut self) -> &MTLBuffer {
&self.buffer
}
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index c48875e70..567507df0 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -1,24 +1,18 @@
-use std::{
- cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,
- rc::Rc,
-};
+use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
-use mpsgraph::{CommandBuffer as MPSCommandBuffer, Device as MPSDevice, Graph};
-use objc2::rc::{Retained, autoreleasepool};
+use metal::CommandBuffer;
use super::{
KernelDataType, MTLContext, ModelShape,
compilation_parameters::CompilationConfig,
encodable_block::{
- ClassifierLayer, ClassifierPredictionHead, Normalization, Pooling,
- Rope,
+ Activation, ClassifierLayer, ClassifierPredictionHead, Normalization,
+ Pooling, Rope,
transformer_layer::{embed_block, linear_block},
},
forward_pass::{
- ArrayId, EncodableBlock, IOArrays, MPSGraphBlock, RopeType,
- ScratchBuffers, SharedBuffers,
+ ArrayId, EncodableBlock, RopeType, ScratchBuffers, SharedBuffers,
},
- graph::{common::activation, placeholder, shaped_type},
kernel::{PoolingKernel, SigmoidKernel},
};
use crate::{
@@ -31,7 +25,7 @@ use crate::{
pub struct ClassifierContext {
pub mtl_context: Rc<MTLContext>,
- pub command_buffer: Retained<MPSCommandBuffer>,
+ pub command_buffer: CommandBuffer,
pub shared_buffers: Rc<RefCell<SharedBuffers>>,
pub scratch_buffers: ScratchBuffers,
@@ -59,8 +53,7 @@ impl ClassifierContext {
let mtl_command_queue =
mtl_device.new_command_queue_with_max_command_buffer_count(1024);
- let command_buffer =
- MPSCommandBuffer::from_command_queue(&mtl_command_queue);
+ let command_buffer = mtl_command_queue.new_command_buffer().to_owned();
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -137,12 +130,8 @@ impl ClassifierContext {
.activation_precision()
.into();
- let embed = embed_block(
- &decoder_config,
- &mtl_context,
- &compilation_config.descriptor_general,
- &root_loader_view,
- );
+ let embed =
+ embed_block(&decoder_config, &mtl_context, &root_loader_view);
let global_rope =
Self::create_rope_block(&mtl_context, data_type, RopeType::Global)
@@ -315,50 +304,22 @@ impl ClassifierContext {
&prediction_head_dense_tree,
ArrayId::ClassifierPooling,
ArrayId::ClassifierPredictionHeadDense,
- &compilation_config.descriptor_general,
);
- let prediction_head_activation = autoreleasepool(|_| {
- let graph = Graph::new();
- let input_shape = [-1, model_dim as isize];
- let input_placeholder =
- placeholder(&graph, &input_shape, prediction_head_data_type);
-
- let output = activation(
- &graph,
- &prediction_head_config.activation,
- &input_placeholder,
+ let prediction_head_activation = Box::new(
+ Activation::new(
+ &mtl_context,
prediction_head_data_type,
- );
-
- let shaped_type_obj =
- shaped_type(&input_shape, prediction_head_data_type);
- let feeds =
- HashMap::from([(&*input_placeholder, &*shaped_type_obj)]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&mtl_context.device),
- &feeds,
- &[&output],
- None,
- Some(&compilation_config.descriptor_general),
- );
-
- let arguments = IOArrays::new(
- vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),
- vec![ArrayId::ClassifierPredictionHeadDense].into_boxed_slice(),
- );
-
- let execution_descriptor =
- mpsgraph::ExecutableExecutionDescriptor::new();
- execution_descriptor.set_enable_commit_and_continue(true);
-
- Box::new(MPSGraphBlock::new(
- executable,
- execution_descriptor,
- arguments,
- )) as Box<dyn EncodableBlock>
- });
+ prediction_head_config.activation.clone(),
+ ArrayId::ClassifierPredictionHeadDense,
+ ArrayId::ClassifierPredictionHeadDense,
+ )
+ .map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(
+ format!("prediction head activation: {:?}", e),
+ ))
+ })?,
+ );
let prediction_head_norm_tree =
prediction_head_tree.subtree("norm").map_err(|_| {
@@ -396,7 +357,6 @@ impl ClassifierContext {
&prediction_head_readout_tree,
ArrayId::ClassifierPredictionHeadNorm,
ArrayId::ClassifierPredictionHeadLogits,
- &compilation_config.descriptor_general,
)
.map_err(|e| {
Error::Classifier(ClassifierError::KernelCreationFailed(format!(
@@ -474,8 +434,7 @@ impl ClassifierContext {
}
pub fn reset_command_buffer(&mut self) {
- self.command_buffer = MPSCommandBuffer::from_command_queue(
- &self.mtl_context.command_queue,
- );
+ self.command_buffer =
+ self.mtl_context.command_queue.new_command_buffer().to_owned();
}
}
diff --git a/crates/uzu/src/backends/metal/compilation_parameters.rs b/crates/uzu/src/backends/metal/compilation_parameters.rs
index 3674ea12b..1bae0ea58 100644
--- a/crates/uzu/src/backends/metal/compilation_parameters.rs
+++ b/crates/uzu/src/backends/metal/compilation_parameters.rs
@@ -1,73 +1,8 @@
-use mpsgraph::{
- CompilationDescriptor, ComputeDevice, Optimization, OptimizationProfile,
-};
-use objc2::rc::Retained;
-
#[derive(Debug, Clone, Copy)]
-pub enum BlockDevice {
- Ane,
- Gpu,
-}
-
-pub fn make_compilation_descriptor(
- device: BlockDevice,
- optimization_level: Optimization,
- optimization_profile: OptimizationProfile,
- preform_placement_analysis: bool,
-) -> Retained<CompilationDescriptor> {
- let desc = CompilationDescriptor::new();
- desc.set_optimization_level(optimization_level);
- desc.set_optimization_profile(optimization_profile);
- desc.set_print_ane_placement_analysis(preform_placement_analysis);
- match device {
- BlockDevice::Ane => {
- desc.set_preferred_device(ComputeDevice::ANE);
- desc.set_allowed_compute_devices(
- ComputeDevice::ANE | ComputeDevice::GPU,
- );
- },
- BlockDevice::Gpu => {
- desc.set_preferred_device(ComputeDevice::GPU);
- desc.set_allowed_compute_devices(ComputeDevice::GPU);
- },
- }
- desc
-}
-
-pub struct CompilationConfig {
- pub descriptor_general: Retained<CompilationDescriptor>,
- pub descriptor_mlp: Retained<CompilationDescriptor>,
-}
+pub struct CompilationConfig {}
impl CompilationConfig {
- fn new(
- descriptor_general: Retained<CompilationDescriptor>,
- descriptor_mlp: Retained<CompilationDescriptor>,
- ) -> Self {
- Self {
- descriptor_general,
- descriptor_mlp,
- }
- }
-
pub fn default() -> Self {
- let optimization_level = Optimization::Level1;
- let optimization_profile = OptimizationProfile::Performance;
- let perform_placement_analysis = false;
-
- Self::new(
- make_compilation_descriptor(
- BlockDevice::Gpu,
- optimization_level,
- optimization_profile,
- perform_placement_analysis,
- ),
- make_compilation_descriptor(
- BlockDevice::Gpu,
- optimization_level,
- optimization_profile,
- perform_placement_analysis,
- ),
- )
+ Self {}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
new file mode 100644
index 000000000..e878e9ce1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/activation/mod.rs
@@ -0,0 +1,86 @@
+use metal::{CommandBufferRef, ComputeCommandEncoderRef};
+
+use super::{
+ super::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::activation::ActivationKernel,
+ },
+ EncodableBlock, EncodingParameters,
+};
+use crate::{Array, DataType, config::Activation as ActivationConfig};
+
+pub struct Activation {
+ kernel: ActivationKernel,
+ config: ActivationConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+}
+
+impl Activation {
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ config: ActivationConfig,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, MTLError> {
+ let kernel = ActivationKernel::new(context, data_type)?;
+ Ok(Self {
+ kernel,
+ config,
+ input_array_id,
+ output_array_id,
+ })
+ }
+}
+
+impl EncodableBlock for Activation {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBufferRef,
+ _parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let mut input_array = arrays[0].borrow_mut();
+ let mut output_array = arrays[1].borrow_mut();
+
+ let n = input_array.shape().iter().product();
+
+ let input_buffer = unsafe { input_array.mtl_buffer() };
+ let output_buffer = unsafe { output_array.mtl_buffer() };
+
+ let encoder = command_buffer.new_compute_command_encoder();
+
+ self.kernel
+ .encode(encoder, &self.config, input_buffer, output_buffer, n)
+ .expect("Failed to encode activation kernel");
+
+ encoder.end_encoding();
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ _parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let mut input_array = arrays[0].borrow_mut();
+ let mut output_array = arrays[1].borrow_mut();
+
+ let n = input_array.shape().iter().product();
+
+ let input_buffer = unsafe { input_array.mtl_buffer() };
+ let output_buffer = unsafe { output_array.mtl_buffer() };
+
+ self.kernel
+ .encode(encoder, &self.config, input_buffer, output_buffer, n)
+ .expect("Failed to encode activation kernel");
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
index 91137f568..226465876 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -1,6 +1,6 @@
//! Attention kernel encodable.
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters};
use crate::backends::metal::{
@@ -48,7 +48,7 @@ impl EncodableBlock for Attention {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let (
@@ -145,9 +145,7 @@ impl EncodableBlock for Attention {
let qkv_buffer = unsafe { qkv_array.mtl_buffer() };
// Prepare command encoder early (used for optional value extraction)
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+ let compute_encoder = command_buffer.new_compute_command_encoder();
// Get KV cache buffers only if KV cache exists (LLM mode)
let has_kv_cache = state.cache_layers().is_some();
@@ -331,10 +329,5 @@ impl EncodableBlock for Attention {
}
compute_encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
index fb9ab8e9f..298cb7849 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -1,6 +1,6 @@
use std::rc::Rc;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use objc2::rc::autoreleasepool;
use super::{
@@ -40,7 +40,7 @@ impl ClassifierLayer {
pub fn new(
mtl_context: &MTLContext,
layer_config: &TransformerLayerConfig,
- compilation_config: Rc<CompilationConfig>,
+ _compilation_config: Rc<CompilationConfig>,
layer_index: usize,
model_dim: usize,
hidden_dim: usize,
@@ -112,7 +112,6 @@ impl ClassifierLayer {
&layer_loader.subtree("mixer.qkv_projection").unwrap(),
ArrayId::Main,
ArrayId::QKV,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create qkv projection");
@@ -151,7 +150,6 @@ impl ClassifierLayer {
&layer_loader.subtree("mixer.out_projection").unwrap(),
ArrayId::AttentionOutput,
ArrayId::Main,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create out projection");
@@ -210,7 +208,6 @@ impl ClassifierLayer {
hidden_dim,
mtl_context,
&layer_loader.subtree("mlp").unwrap(),
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create mlp block");
@@ -278,7 +275,7 @@ impl EncodableBlock for ClassifierLayer {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
#[cfg(feature = "tracing")]
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
index c05f7b2f3..481ca2559 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -2,7 +2,7 @@
use std::rc::Rc;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{
EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,
@@ -40,19 +40,11 @@ impl Decoder {
.subtree("transformer")
.expect("transformer subtree not found");
- let embed = embed_block(
- &decoder_config,
- &mtl_context,
- &compilation_config.descriptor_general,
- root_weight_loader,
- );
+ let embed =
+ embed_block(&decoder_config, &mtl_context, root_weight_loader);
- let readout = readout_block(
- &decoder_config,
- &mtl_context,
- &compilation_config.descriptor_general,
- root_weight_loader,
- );
+ let readout =
+ readout_block(&decoder_config, &mtl_context, root_weight_loader);
let attention_data_type = Self::attention_data_type(&decoder_config);
let norm_reference_layer = decoder_config
@@ -209,7 +201,7 @@ impl EncodableBlock for Decoder {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let mut resolver = EncoderResolver::new(command_buffer);
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs
index c13db8c9a..da85f713d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs
@@ -1,7 +1,7 @@
use crate::{DataType, backends::metal::MTLError};
#[derive(Debug, thiserror::Error)]
-pub enum QuantizedEmbeddingError {
+pub enum EmbeddingError {
#[error("Metal error: {0}")]
MetalError(#[from] MTLError),
#[error("Unsupported data type: {0:?}")]
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
index 5bd6c6ae1..527c1febc 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
@@ -1,9 +1,11 @@
use std::rc::Rc;
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef};
-use super::QuantizedEmbeddingError;
+use super::{
+ super::{EncodableBlock, EncodingParameters},
+ EmbeddingError,
+};
use crate::{
Array, DataType,
backends::metal::{
@@ -17,8 +19,6 @@ use crate::{
parameters::ParameterTree,
};
-use super::super::{EncodableBlock, EncodingParameters};
-
pub struct FullPrecisionEmbeddingLookup {
kernel: FullPrecisionEmbeddingLookupKernel,
weights_buffer: MTLBuffer,
@@ -35,14 +35,14 @@ impl FullPrecisionEmbeddingLookup {
model_dim: usize,
input_scale: Option<f32>,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
let kernel =
FullPrecisionEmbeddingLookupKernel::new(mtl_context, data_type)?;
let mut weights = match parameter_tree.leaf("weights") {
Ok(weights) => weights,
Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ EmbeddingError::MetalError(MTLError::Generic(format!(
"Failed to load weights: {:?}",
e
)))
@@ -50,7 +50,7 @@ impl FullPrecisionEmbeddingLookup {
};
if weights.shape() != [vocab_size, model_dim] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding lookup weights shape mismatch: got {:?}, \
expected [{}, {}]",
@@ -62,7 +62,7 @@ impl FullPrecisionEmbeddingLookup {
}
if weights.data_type() != data_type {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Weights dtype mismatch: got {:?}, expected {:?}",
weights.data_type(),
@@ -87,7 +87,7 @@ impl EncodableBlock for FullPrecisionEmbeddingLookup {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
@@ -98,8 +98,7 @@ impl EncodableBlock for FullPrecisionEmbeddingLookup {
let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let args = FullPrecisionEmbeddingLookupArguments {
token_ids_buffer,
@@ -118,11 +117,8 @@ impl EncodableBlock for FullPrecisionEmbeddingLookup {
encoder.end_encoding();
if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
-
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
index b3c6c0860..9d373f2ca 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
@@ -1,11 +1,10 @@
use std::{cell::RefCell, rc::Rc};
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef};
use super::{
super::{EncodableBlock, EncodingParameters},
- QuantizedEmbeddingError,
+ EmbeddingError,
};
use crate::{
Array, DataType,
@@ -31,10 +30,10 @@ impl FullPrecisionEmbeddingReadout {
vocab_size: usize,
model_dim: usize,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
{
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ return Err(EmbeddingError::UnsupportedDataType(
data_type,
));
}
@@ -42,7 +41,7 @@ impl FullPrecisionEmbeddingReadout {
let mut weights = match parameter_tree.leaf("weights") {
Ok(weights) => weights,
Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ EmbeddingError::MetalError(MTLError::Generic(format!(
"Failed to load weights: {:?}",
e
)))
@@ -50,7 +49,7 @@ impl FullPrecisionEmbeddingReadout {
};
if weights.shape() != [vocab_size, model_dim] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
weights.shape(),
@@ -61,7 +60,7 @@ impl FullPrecisionEmbeddingReadout {
}
if weights.data_type() != data_type {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Weights dtype mismatch: got {:?}, expected {:?}",
weights.data_type(),
@@ -75,7 +74,7 @@ impl FullPrecisionEmbeddingReadout {
// Weights are [vocab_size, model_dim], we compute input @ weights^T
// MatmulKernel with transpose_b=true handles this
let kernel = MatmulKernel::new(mtl_context, data_type, false, true)
- .map_err(QuantizedEmbeddingError::MetalError)?;
+ .map_err(EmbeddingError::MetalError)?;
Ok(Self {
kernel: RefCell::new(kernel),
@@ -90,7 +89,7 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
@@ -101,8 +100,7 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {
let input_buffer = unsafe { input_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let args = MatmulArguments {
a: input_buffer,
@@ -125,10 +123,8 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {
encoder.end_encoding();
if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
index 3c1386abd..e376c571b 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
@@ -6,7 +6,7 @@ mod full_precision_readout;
mod quantized_lookup;
mod quantized_readout;
-pub use error::QuantizedEmbeddingError;
+pub use error::EmbeddingError;
pub use full_precision_lookup::FullPrecisionEmbeddingLookup;
pub use full_precision_readout::FullPrecisionEmbeddingReadout;
pub use quantized_lookup::QuantizedEmbeddingLookup;
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
index 4c1510b2a..9f4099dc3 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
@@ -1,11 +1,10 @@
use std::rc::Rc;
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef};
use super::{
super::{EncodableBlock, EncodingParameters},
- QuantizedEmbeddingError,
+ EmbeddingError,
};
use crate::{
Array, DataType,
@@ -39,7 +38,7 @@ impl QuantizedEmbeddingLookup {
group_size: usize,
mode: QuantizationMode,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
Self::new_with_names(
mtl_context,
data_type,
@@ -62,7 +61,7 @@ impl QuantizedEmbeddingLookup {
group_size: usize,
mode: QuantizationMode,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
Self::new_with_names(
mtl_context,
data_type,
@@ -88,7 +87,7 @@ impl QuantizedEmbeddingLookup {
scales_name: &str,
biases_name: &str,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
let packing_divisor = mode.packing_divisor();
let kernel =
@@ -96,14 +95,14 @@ impl QuantizedEmbeddingLookup {
// Load weights [vocab_size, model_dim/packing_divisor] as storage_type
let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ EmbeddingError::MetalError(MTLError::Generic(format!(
"Failed to load weights: {:?}",
e
)))
})?;
if weights.data_type() != mode.storage_type() {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Expected packed weights of type {:?}, got {:?}",
mode.storage_type(),
@@ -114,7 +113,7 @@ impl QuantizedEmbeddingLookup {
// Load scales [vocab_size, num_groups]
let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ EmbeddingError::MetalError(MTLError::Generic(format!(
"Failed to load scales: {:?}",
e
)))
@@ -123,7 +122,7 @@ impl QuantizedEmbeddingLookup {
// Validate shapes and types
let num_groups = (model_dim + group_size - 1) / group_size;
if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
weights.shape(),
@@ -133,7 +132,7 @@ impl QuantizedEmbeddingLookup {
));
}
if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
scales.shape(),
@@ -143,7 +142,7 @@ impl QuantizedEmbeddingLookup {
));
}
if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ return Err(EmbeddingError::UnsupportedDataType(
scales.data_type(),
));
}
@@ -152,7 +151,7 @@ impl QuantizedEmbeddingLookup {
let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {
Ok(mut deq_biases) => {
if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",
deq_biases.shape(),
@@ -162,7 +161,7 @@ impl QuantizedEmbeddingLookup {
));
}
if deq_biases.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ return Err(EmbeddingError::UnsupportedDataType(
deq_biases.data_type(),
));
}
@@ -174,7 +173,7 @@ impl QuantizedEmbeddingLookup {
DataType::F32 => 4,
other => {
return Err(
- QuantizedEmbeddingError::UnsupportedDataType(other),
+ EmbeddingError::UnsupportedDataType(other),
);
},
};
@@ -213,7 +212,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
@@ -224,8 +223,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {
let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let args = QuantizedEmbeddingLookupArguments {
token_ids_buffer,
@@ -246,10 +244,8 @@ impl EncodableBlock for QuantizedEmbeddingLookup {
encoder.end_encoding();
if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
index 6b619ae93..064ed8ca7 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
@@ -1,11 +1,10 @@
use std::rc::Rc;
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef};
use super::{
super::{EncodableBlock, EncodingParameters},
- QuantizedEmbeddingError,
+ EmbeddingError,
};
use crate::{
Array, DataType,
@@ -38,7 +37,7 @@ impl QuantizedEmbeddingReadout {
group_size: usize,
mode: QuantizationMode,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
Self::new_with_names(
mtl_context,
data_type,
@@ -61,7 +60,7 @@ impl QuantizedEmbeddingReadout {
group_size: usize,
mode: QuantizationMode,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
Self::new_with_names(
mtl_context,
data_type,
@@ -87,10 +86,10 @@ impl QuantizedEmbeddingReadout {
scales_name: &str,
biases_name: &str,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
// Load weights [vocab_size, model_dim/2] as U8
let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ EmbeddingError::MetalError(MTLError::Generic(format!(
"Failed to load weights: {:?}",
e
)))
@@ -98,7 +97,7 @@ impl QuantizedEmbeddingReadout {
// Load scales [vocab_size, num_groups]
let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ EmbeddingError::MetalError(MTLError::Generic(format!(
"Failed to load scales: {:?}",
e
)))
@@ -112,7 +111,7 @@ impl QuantizedEmbeddingReadout {
let weights_transposed = weights.shape()[0] == vocab_size;
if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
weights.shape(),
@@ -122,7 +121,7 @@ impl QuantizedEmbeddingReadout {
));
}
if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
scales.shape(),
@@ -132,7 +131,7 @@ impl QuantizedEmbeddingReadout {
));
}
if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ return Err(EmbeddingError::UnsupportedDataType(
scales.data_type(),
));
}
@@ -141,7 +140,7 @@ impl QuantizedEmbeddingReadout {
let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {
Ok(mut deq_biases) => {
if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
+ return Err(EmbeddingError::MetalError(
MTLError::Generic(format!(
"Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
deq_biases.shape(),
@@ -151,7 +150,7 @@ impl QuantizedEmbeddingReadout {
));
}
if deq_biases.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ return Err(EmbeddingError::UnsupportedDataType(
deq_biases.data_type(),
));
}
@@ -164,7 +163,7 @@ impl QuantizedEmbeddingReadout {
DataType::F32 => 4,
other => {
return Err(
- QuantizedEmbeddingError::UnsupportedDataType(other),
+ EmbeddingError::UnsupportedDataType(other),
);
},
};
@@ -198,7 +197,7 @@ impl QuantizedEmbeddingReadout {
weights_transposed,
)
.map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ EmbeddingError::MetalError(MTLError::Generic(format!(
"Failed to create kernel: {:?}",
e
)))
@@ -219,7 +218,7 @@ impl EncodableBlock for QuantizedEmbeddingReadout {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
@@ -230,8 +229,7 @@ impl EncodableBlock for QuantizedEmbeddingReadout {
let input_buffer = unsafe { input_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let args = QuantizedMatmulArguments {
a_buffer: input_buffer,
@@ -252,10 +250,8 @@ impl EncodableBlock for QuantizedEmbeddingReadout {
encoder.end_encoding();
if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index a2cd5dc3a..70fdc92cc 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -2,7 +2,7 @@
use std::rc::Rc;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use objc2::rc::autoreleasepool;
use super::{
@@ -114,7 +114,6 @@ impl LayerExecutables {
.unwrap(),
ArrayId::Main,
ArrayId::QKV,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create qkv projection");
@@ -155,7 +154,6 @@ impl LayerExecutables {
.unwrap(),
ArrayId::AttentionOutput,
ArrayId::Main,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create out projection");
@@ -263,7 +261,6 @@ impl LayerExecutables {
hidden_dim,
mtl_context,
&decoder_layer_loader.subtree("mlp").unwrap(),
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create mlp block");
@@ -305,7 +302,7 @@ impl EncodableBlock for LayerExecutables {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
#[cfg(feature = "tracing")]
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
index 155324388..c8bf57461 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -1,7 +1,6 @@
use std::{cell::RefCell, rc::Rc};
-use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef};
use crate::{
DataType,
@@ -117,7 +116,7 @@ impl EncodableBlock for FullPrecisionLinear {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
@@ -129,8 +128,7 @@ impl EncodableBlock for FullPrecisionLinear {
let input_buffer = unsafe { input_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let args = MatmulArguments {
a: input_buffer,
@@ -156,24 +154,16 @@ impl EncodableBlock for FullPrecisionLinear {
(&self.bias_add_kernel, &self.biases_buffer)
{
let total_len = batch_size * self.output_dim;
- let retained_cb = root_command_buffer.to_owned();
bias_add.encode_into_command_buffer(
&output_buffer,
bias_buf,
&output_buffer,
self.output_dim,
total_len,
- &retained_cb,
+ command_buffer,
parameters.predicate,
);
}
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
}
fn supports_shared_encoder(&self) -> bool {
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
index 9fab0314e..1027bab77 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
@@ -1,7 +1,6 @@
use std::rc::Rc;
-use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef};
use crate::{
DataType,
@@ -226,7 +225,7 @@ impl EncodableBlock for QuantizedLinear {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
@@ -238,8 +237,7 @@ impl EncodableBlock for QuantizedLinear {
let input_buffer = unsafe { input_array_mut.mtl_buffer() };
let output_buffer = unsafe { output_array_mut.mtl_buffer() };
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let args = QuantizedMatmulArguments {
a_buffer: input_buffer,
@@ -263,23 +261,20 @@ impl EncodableBlock for QuantizedLinear {
(&self.bias_add_kernel, &self.biases_buffer)
{
let total_len = batch_size * self.output_dim;
- let retained_cb = root_command_buffer.to_owned();
bias_add.encode_into_command_buffer(
&output_buffer,
bias_buf,
&output_buffer,
self.output_dim,
total_len,
- &retained_cb,
+ command_buffer,
parameters.predicate,
);
}
if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index 047656f42..d2e80789f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -2,7 +2,7 @@
use std::{env, rc::Rc};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters, transformer_layer};
use crate::{
@@ -44,7 +44,7 @@ impl MambaMixer {
mtl_context: &MTLContext,
layer_type: DecoderLayerType,
mamba_config: Mamba2Config,
- compilation_config: Rc<CompilationConfig>,
+ _compilation_config: Rc<CompilationConfig>,
layer_index: usize,
model_dim: usize,
num_heads: usize,
@@ -82,7 +82,6 @@ impl MambaMixer {
),
ArrayId::Main,
ArrayId::SsmInProj,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create in-projection kernel");
@@ -98,7 +97,6 @@ impl MambaMixer {
),
ArrayId::AttentionOutput,
ArrayId::Main,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create out-projection kernel");
@@ -147,7 +145,7 @@ impl MambaMixer {
fn encode_pipeline(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let active_suffix_length = state.active_suffix_length();
@@ -168,17 +166,15 @@ impl MambaMixer {
self.out_projection.encode(state, command_buffer, parameters);
if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
fn run_split_inproj(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -204,9 +200,7 @@ impl MambaMixer {
let num_heads = self.config.num_heads;
let total_dim = conv_dim + inner_dim + num_heads;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute = mtl_command_buffer.new_compute_command_encoder();
+ let compute = command_buffer.new_compute_command_encoder();
self.split_inproj
.encode(
&compute,
@@ -230,7 +224,7 @@ impl MambaMixer {
fn run_conv_scan(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -262,7 +256,7 @@ impl MambaMixer {
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let proj_dim = self.config.num_groups * self.config.state_dim;
- let cmd = command_buffer.root_command_buffer().to_owned();
+ let cmd = command_buffer.to_owned();
let state_stride = self.config.kernel_size.saturating_sub(1);
drop(conv_state);
@@ -355,7 +349,7 @@ impl MambaMixer {
fn run_prefill_ssm(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
suffix_length: usize,
) {
let base_arrays = state.arrays(&[
@@ -392,7 +386,7 @@ impl MambaMixer {
let mut skip_weights = self.skip_connection_weight.clone();
let skip = unsafe { skip_weights.mtl_buffer().to_owned() };
- let cmd = command_buffer.root_command_buffer().to_owned();
+ let cmd = command_buffer.to_owned();
let compute = cmd.new_compute_command_encoder();
self.ssm_prefill
.encode(
@@ -438,7 +432,7 @@ impl MambaMixer {
fn run_decode_ssm(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -485,7 +479,7 @@ impl MambaMixer {
let group_size = (h / g) as i32;
let state_size = n as i32;
- let cmd = command_buffer.root_command_buffer().to_owned();
+ let cmd = command_buffer.to_owned();
let compute = cmd.new_compute_command_encoder();
self.ssd_update
.encode(
@@ -533,7 +527,7 @@ impl EncodableBlock for MambaMixer {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
self.encode_pipeline(state, command_buffer, parameters);
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
index 3b9c104fe..e3cbf1a28 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -1,6 +1,6 @@
//! MLP block encodable.
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters, QuantizedLinear};
use crate::backends::metal::{
@@ -32,7 +32,7 @@ impl EncodableBlock for MlpBlock {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
params: &EncodingParameters,
) {
// Up
@@ -42,8 +42,7 @@ impl EncodableBlock for MlpBlock {
let mut fused = arrays[0].borrow_mut();
let mut hidden = arrays[1].borrow_mut();
let active_suffix_length = state.active_suffix_length() as i32;
- let root = command_buffer.root_command_buffer();
- let encoder = root.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let fused_buf = unsafe { fused.mtl_buffer() };
let hidden_buf = unsafe { hidden.mtl_buffer() };
self.gate
@@ -56,10 +55,8 @@ impl EncodableBlock for MlpBlock {
self.down.encode(state, command_buffer, params);
if params.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 43ff39948..ea80d2c61 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -6,11 +6,11 @@
//! Encodables implement `EncodableBlock` and orchestrate one or more
//! kernels to perform operations on `ForwardPassState`.
-use metal::ComputeCommandEncoderRef;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::{CommandBufferRef, ComputeCommandEncoderRef};
use super::forward_pass::ForwardPassState;
+mod activation;
mod attention;
mod classifier_layer;
mod decoder;
@@ -31,12 +31,14 @@ mod tensor_add_swap;
mod tensor_copy;
pub mod transformer_layer;
+pub use activation::Activation;
pub use attention::Attention;
pub use classifier_layer::ClassifierLayer;
pub use decoder::Decoder;
pub use embedding::{
- FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout,
- QuantizedEmbeddingError, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,
+ EmbeddingError, FullPrecisionEmbeddingLookup,
+ FullPrecisionEmbeddingReadout, QuantizedEmbeddingLookup,
+ QuantizedEmbeddingReadout,
};
pub use encoding_parameters::EncodingParameters;
pub use layer::LayerExecutables;
@@ -60,7 +62,7 @@ pub trait EncodableBlock {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
);
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
index ec45d94b4..35b9ec7c8 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -2,7 +2,7 @@
use std::{cell::RefCell, rc::Rc};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{
super::{EncodableBlock, EncodingParameters},
@@ -291,7 +291,7 @@ impl EncodableBlock for MoeBlock {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let suffix_length = state.active_suffix_length();
@@ -350,8 +350,7 @@ impl EncodableBlock for MoeBlock {
let e = self.moe_config.mixture_size;
let k = self.moe_config.num_experts_per_token;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
+ let mtl_command_buffer = command_buffer.to_owned();
let root = &*mtl_command_buffer;
let k_tile = 128;
@@ -626,8 +625,8 @@ impl EncodableBlock for MoeBlock {
.expect("MoE finalize failed");
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
index a9e2b56a5..af9997007 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/layer_norm.rs
@@ -2,8 +2,7 @@
use std::rc::Rc;
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef};
use super::super::{EncodableBlock, EncodingParameters};
use crate::{
@@ -89,7 +88,7 @@ impl EncodableBlock for LayerNorm {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let input_binding = state.arrays(&[self.input_array_id]);
@@ -109,9 +108,7 @@ impl EncodableBlock for LayerNorm {
let batch_size = input_shape[0] as i32;
let model_dim = input_shape[1] as i32;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+ let compute_encoder = command_buffer.new_compute_command_encoder();
if let Err(e) = self.kernel.encode(
&compute_encoder,
@@ -131,8 +128,8 @@ impl EncodableBlock for LayerNorm {
compute_encoder.end_encoding();
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
index d2d7309cf..01c3e6ff5 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/mod.rs
@@ -7,7 +7,7 @@ mod rms_norm;
use std::rc::Rc;
pub use layer_norm::LayerNorm;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
pub use qk_norm::QKNorm;
pub use rms_norm::RMSNorm;
@@ -77,7 +77,7 @@ impl EncodableBlock for Normalization {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
match self {
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
index c8e39f808..8d7f9f7cc 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/qk_norm.rs
@@ -2,8 +2,7 @@
use std::rc::Rc;
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef};
use super::super::{EncodableBlock, EncodingParameters};
use crate::{
@@ -163,7 +162,7 @@ impl EncodableBlock for QKNorm {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let qkv_binding = state.arrays(&[self.qkv_array_id]);
@@ -178,9 +177,7 @@ impl EncodableBlock for QKNorm {
let batch_size = qkv_shape[0] as i32;
let head_dim = self.head_dim as i32;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+ let compute_encoder = command_buffer.new_compute_command_encoder();
// Process query normalization if configured
if let (
@@ -240,8 +237,8 @@ impl EncodableBlock for QKNorm {
compute_encoder.end_encoding();
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
index 83a96877e..d1ea6d1cb 100644
--- a/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/normalization/rms_norm.rs
@@ -2,8 +2,7 @@
use std::rc::Rc;
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::{Buffer as MTLBuffer, CommandBufferRef};
use super::super::{EncodableBlock, EncodingParameters};
use crate::{
@@ -93,7 +92,7 @@ impl EncodableBlock for RMSNorm {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let input_binding = state.arrays(&[self.input_array_id]);
@@ -113,9 +112,7 @@ impl EncodableBlock for RMSNorm {
let batch_size = input_shape[0] as i32;
let model_dim = input_shape[1] as i32;
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+ let compute_encoder = command_buffer.new_compute_command_encoder();
if let Err(e) = self.kernel.encode(
&compute_encoder,
@@ -135,8 +132,8 @@ impl EncodableBlock for RMSNorm {
compute_encoder.end_encoding();
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/pooling.rs b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
index 323585f66..deb420f0b 100644
--- a/crates/uzu/src/backends/metal/encodable_block/pooling.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/pooling.rs
@@ -1,6 +1,6 @@
//! Pooling encodable for sequence-level aggregation.
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters};
use crate::{
@@ -36,7 +36,7 @@ impl EncodableBlock for Pooling {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
_parameters: &EncodingParameters,
) {
let batch_size = 1;
@@ -51,8 +51,7 @@ impl EncodableBlock for Pooling {
let mut pooling_array = arrays[1].borrow_mut();
let output_buffer = unsafe { pooling_array.mtl_buffer().to_owned() };
- let root = command_buffer.root_command_buffer();
- let encoder = root.new_compute_command_encoder();
+ let encoder = command_buffer.new_compute_command_encoder();
let result = match self.pooling_type {
PoolingType::Cls => self.pooling_kernel.encode_cls(
@@ -81,8 +80,6 @@ impl EncodableBlock for Pooling {
drop(main_array);
drop(pooling_array);
- command_buffer.commit_and_continue();
-
#[cfg(feature = "tracing")]
{
let traces_rc = state.traces().clone();
@@ -92,8 +89,7 @@ impl EncodableBlock for Pooling {
drop(trace_arr);
drop(traces_ref);
- let root = command_buffer.root_command_buffer();
- let blit = root.new_blit_command_encoder();
+ let blit = command_buffer.new_blit_command_encoder();
blit.copy_from_buffer(
&output_buffer,
0,
@@ -103,10 +99,6 @@ impl EncodableBlock for Pooling {
as u64,
);
blit.end_encoding();
-
- let root_owned = root.to_owned();
- command_buffer.commit_and_continue();
- root_owned.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
index 25a4fcf4f..3f468746a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/prediction_head.rs
@@ -1,6 +1,6 @@
//! Prediction head encodable for classification output.
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters};
#[cfg(feature = "tracing")]
@@ -40,7 +40,7 @@ impl EncodableBlock for ClassifierPredictionHead {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
self.dense.encode(state, command_buffer, parameters);
@@ -49,11 +49,15 @@ impl EncodableBlock for ClassifierPredictionHead {
self.readout.encode(state, command_buffer, parameters);
#[cfg_attr(feature = "tracing", allow(unused_variables))]
- let root_after_head = command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
+ let root_after_head = command_buffer.to_owned();
+ if parameters.wait_until_completed {
+ command_buffer.commit();
+ }
#[cfg(not(feature = "tracing"))]
- root_after_head.wait_until_completed();
+ if parameters.wait_until_completed {
+ root_after_head.wait_until_completed();
+ }
#[cfg(feature = "tracing")]
{
@@ -77,8 +81,7 @@ impl EncodableBlock for ClassifierPredictionHead {
(batch_size * self.num_labels * data_type.size_in_bytes())
as u64;
- let root = command_buffer.root_command_buffer();
- let blit = root.new_blit_command_encoder();
+ let blit = command_buffer.new_blit_command_encoder();
blit.copy_from_buffer(
&linear_output_buffer,
0,
@@ -88,8 +91,8 @@ impl EncodableBlock for ClassifierPredictionHead {
);
blit.end_encoding();
- let root_owned = root.to_owned();
- command_buffer.commit_and_continue();
+ let root_owned = command_buffer.to_owned();
+ command_buffer.commit();
root_owned.wait_until_completed();
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/rope.rs b/crates/uzu/src/backends/metal/encodable_block/rope.rs
index 94b8731da..4163fa962 100644
--- a/crates/uzu/src/backends/metal/encodable_block/rope.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/rope.rs
@@ -1,6 +1,6 @@
//! Rope (Rotary Position Embedding) encodable.
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters};
use crate::{
@@ -35,7 +35,7 @@ impl EncodableBlock for Rope {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let (suffix_length, num_heads, head_dim, num_groups, rope_max_seq_len) = {
@@ -81,9 +81,7 @@ impl EncodableBlock for Rope {
state.arrays(&[ArrayId::RopeSines(self.rope_type)]);
let mut rope_sines = sin_buffer_binding[0].borrow_mut();
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute_encoder = mtl_command_buffer.new_compute_command_encoder();
+ let compute_encoder = command_buffer.new_compute_command_encoder();
let token_positions_offset = token_positions.buffer_offset();
@@ -112,8 +110,8 @@ impl EncodableBlock for Rope {
compute_encoder.end_encoding();
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/sampling.rs b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
index 7b1d21648..50a3ab92f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/sampling.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/sampling.rs
@@ -2,7 +2,7 @@
use std::rc::Rc;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters};
use crate::backends::metal::{
@@ -57,7 +57,7 @@ impl EncodableBlock for Sampling {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
assert!(
@@ -84,8 +84,7 @@ impl EncodableBlock for Sampling {
let sampling_method = state.sampling_method().unwrap();
let seeds_offset = seeds.buffer_offset();
- let root_command_buffer =
- command_buffer.root_command_buffer().to_owned();
+ let root_command_buffer = command_buffer.to_owned();
let bitmask_buffer = state
.token_bitmask()
.map(|cell| unsafe { cell.borrow_mut().mtl_buffer().clone() });
@@ -104,8 +103,8 @@ impl EncodableBlock for Sampling {
}
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- root_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index 8823cf39b..f3c280e2e 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -1,6 +1,6 @@
use std::rc::Rc;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters, transformer_layer};
use crate::{
@@ -47,7 +47,7 @@ impl ShortConvMixer {
mtl_context: &MTLContext,
layer_type: DecoderLayerType,
short_conv_config: ShortConvConfig,
- compilation_config: Rc<CompilationConfig>,
+ _compilation_config: Rc<CompilationConfig>,
layer_index: usize,
model_dim: usize,
decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,
@@ -77,7 +77,6 @@ impl ShortConvMixer {
&resolve_subtree(&mixer_tree, &["in_projection", "in_proj"]),
ArrayId::Main,
ArrayId::SsmInProj,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create in-projection kernel");
@@ -90,7 +89,6 @@ impl ShortConvMixer {
&resolve_subtree(&mixer_tree, &["out_projection", "out_proj"]),
ArrayId::AttentionOutput,
ArrayId::Main,
- &compilation_config.descriptor_mlp,
)
.expect("Failed to create out-projection kernel");
@@ -120,7 +118,7 @@ impl ShortConvMixer {
fn encode_pipeline(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let active_suffix_length = state.active_suffix_length();
@@ -139,17 +137,15 @@ impl ShortConvMixer {
self.out_projection.encode(state, command_buffer, parameters);
if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
fn run_prefill_conv(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
suffix_length: usize,
) {
let padded = state
@@ -237,7 +233,7 @@ impl ShortConvMixer {
fn run_decode_conv(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
suffix_length: usize,
) {
let arrays = state.arrays(&[
@@ -263,9 +259,7 @@ impl ShortConvMixer {
let kernel_size = self.config.kernel_size;
let state_stride = kernel_size.saturating_sub(1);
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute = mtl_command_buffer.new_compute_command_encoder();
+ let compute = command_buffer.new_compute_command_encoder();
self.short_conv_kernel
.encode_decode(
@@ -294,7 +288,7 @@ impl EncodableBlock for ShortConvMixer {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
self.encode_pipeline(state, command_buffer, parameters);
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
index ae72e7d47..ba3837b22 100644
--- a/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_add_swap.rs
@@ -1,6 +1,6 @@
//! Tensor add-swap encodable.
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters};
use crate::{
@@ -35,7 +35,7 @@ impl EncodableBlock for TensorAddSwap {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&self.argument_arrays);
@@ -48,19 +48,16 @@ impl EncodableBlock for TensorAddSwap {
let skip_mtl_buffer = unsafe { skip_array.mtl_buffer() };
let main_mtl_buffer = unsafe { main_array.mtl_buffer() };
- let retained_mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
-
self.kernel.encode_into_command_buffer(
&skip_mtl_buffer,
&main_mtl_buffer,
length,
- &retained_mtl_command_buffer,
+ command_buffer,
);
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- retained_mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
index 61156c8fa..c2a090fed 100644
--- a/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/tensor_copy.rs
@@ -1,6 +1,6 @@
//! Tensor copy encodable.
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBufferRef;
use super::{EncodableBlock, EncodingParameters};
use crate::{
@@ -35,7 +35,7 @@ impl EncodableBlock for TensorCopy {
fn encode(
&self,
state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
parameters: &EncodingParameters,
) {
let arrays = state.arrays(&self.argument_arrays);
@@ -48,19 +48,16 @@ impl EncodableBlock for TensorCopy {
let source_mtl_buffer = unsafe { source_array.mtl_buffer() };
let destination_mtl_buffer = unsafe { destination_array.mtl_buffer() };
- let retained_mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
-
self.kernel.encode_into_command_buffer(
&source_mtl_buffer,
&destination_mtl_buffer,
length,
- &retained_mtl_command_buffer,
+ command_buffer,
);
if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- retained_mtl_command_buffer.wait_until_completed();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 27d00f4c1..d26cf38b1 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -1,10 +1,4 @@
-use std::{collections::HashMap, rc::Rc};
-
-use mpsgraph::{
- CompilationDescriptor, Device as MPSDevice, ExecutableExecutionDescriptor,
- Graph,
-};
-use objc2::rc::Retained;
+use std::rc::Rc;
use super::{
EncodableBlock, FullPrecisionEmbeddingLookup,
@@ -14,24 +8,13 @@ use super::{
use crate::{
DataType,
backends::metal::{
- MTLContext, MTLError,
- forward_pass::{ArrayId, IOArrays, MPSGraphBlock},
- graph::{
- embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
- embeddings_readout_subgraph, placeholder, shaped_type,
- },
+ MTLContext, MTLError, forward_pass::ArrayId,
kernel::mlp::MlpGateActMulEncodable,
},
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
parameters::ParameterTree,
};
-fn make_execution_descriptor() -> Retained<ExecutableExecutionDescriptor> {
- let execution_descriptor = ExecutableExecutionDescriptor::new();
- execution_descriptor.set_enable_commit_and_continue(true);
- execution_descriptor
-}
-
pub fn linear_block<const N: usize>(
config: &LinearConfig,
_has_biases: bool,
@@ -41,7 +24,6 @@ pub fn linear_block<const N: usize>(
parameter_tree: &ParameterTree<Rc<MTLContext>>,
input_array_id: ArrayId,
output_array_id: ArrayId,
- _compilation_descriptor: &CompilationDescriptor,
) -> Result<Box<dyn EncodableBlock>, MTLError> {
let out_sum: usize = output_dims.iter().sum();
match config {
@@ -86,7 +68,6 @@ pub fn mlp_block(
hidden_dim: usize,
context: &MTLContext,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- _compilation_descriptor: &CompilationDescriptor,
) -> Result<Box<dyn EncodableBlock>, MTLError> {
if let crate::config::MLPConfig::Dense(dense) = config {
match &dense.linear_config {
@@ -190,7 +171,6 @@ pub fn mlp_block(
pub fn embed_block(
config: &DecoderConfig,
context: &MTLContext,
- compilation_descriptor: &CompilationDescriptor,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Box<dyn EncodableBlock> {
match &config.embedding_config {
@@ -281,74 +261,29 @@ pub fn embed_block(
Box::new(block)
},
EmbeddingConfig::QuantizedTied {
- embedding_quantization_mode: _,
+ embedding_quantization_mode,
+ activation_precision,
..
} => {
- let graph = Graph::new();
-
- let input_shape = [-1 as isize];
- let input_data_type = DataType::U64;
- let input_placeholder =
- placeholder(&graph, &input_shape, input_data_type);
- let input_shaped_type = shaped_type(&input_shape, input_data_type);
- let weights_shape =
- [config.vocab_size as isize, (config.model_dim / 2) as isize];
- let weights_data_type = DataType::U8;
- let weights_shaped_type =
- shaped_type(&weights_shape, weights_data_type);
- let weights_placeholder =
- placeholder(&graph, &weights_shape, weights_data_type);
-
- let scales_shape = [config.vocab_size as isize];
- let scales_data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let scales_shaped_type =
- shaped_type(&scales_shape, scales_data_type);
- let scales_placeholder =
- placeholder(&graph, &scales_shape, scales_data_type);
-
- let weights = embeddings_dequantize_weights_subgraph(
- &graph,
- &weights_placeholder,
- &scales_placeholder,
- )
- .unwrap();
+ let data_type: DataType = (*activation_precision).into();
+
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
- let output = embeddings_embed_subgraph(
- &graph,
- &config.embedding_config,
- &input_placeholder,
- &weights,
+ // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim
+ let group_size = config.model_dim;
+
+ let block = QuantizedEmbeddingLookup::new_tied(
+ context,
+ data_type,
+ config.vocab_size,
+ config.model_dim,
+ group_size,
+ *embedding_quantization_mode,
+ &embeddings_tree,
)
- .unwrap();
-
- let feeds = HashMap::from([
- (&*input_placeholder, &*input_shaped_type),
- (&*weights_placeholder, &*weights_shaped_type),
- (&*scales_placeholder, &*scales_shaped_type),
- ]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(&compilation_descriptor),
- );
-
- let block = MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![
- ArrayId::TokenIds,
- ArrayId::EmbeddingsInputWeights,
- ArrayId::EmbeddingsScales,
- ]
- .into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- ),
- );
+ .expect("Failed to create quantized embedding lookup kernel");
Box::new(block)
},
@@ -414,10 +349,9 @@ pub fn embed_block(
pub fn readout_block(
config: &DecoderConfig,
context: &MTLContext,
- compilation_descriptor: &CompilationDescriptor,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Box<dyn EncodableBlock> {
- match config.embedding_config {
+ match &config.embedding_config {
EmbeddingConfig::Tied {
precision,
..
@@ -428,7 +362,7 @@ pub fn readout_block(
let block = FullPrecisionEmbeddingReadout::new(
context,
- precision.into(),
+ (*precision).into(),
config.vocab_size,
config.model_dim,
&embeddings_tree,
@@ -447,7 +381,7 @@ pub fn readout_block(
let block = FullPrecisionEmbeddingReadout::new(
context,
- precision.into(),
+ (*precision).into(),
config.vocab_size,
config.model_dim,
&embeddings_tree,
@@ -462,7 +396,7 @@ pub fn readout_block(
embedding_quantization_mode,
..
} => {
- let data_type: DataType = activation_precision.into();
+ let data_type: DataType = (*activation_precision).into();
let embeddings_tree = parameter_tree
.subtree("embedding")
.expect("Failed to get embedding subtree");
@@ -472,8 +406,8 @@ pub fn readout_block(
data_type,
config.vocab_size,
config.model_dim,
- group_size,
- embedding_quantization_mode,
+ *group_size,
+ *embedding_quantization_mode,
&embeddings_tree,
)
.expect("Failed to create quantized embedding readout kernel");
@@ -481,75 +415,29 @@ pub fn readout_block(
Box::new(block)
},
EmbeddingConfig::QuantizedTied {
- embedding_quantization_mode: _,
+ embedding_quantization_mode,
+ activation_precision,
..
} => {
- let graph = Graph::new();
+ let data_type: DataType = (*activation_precision).into();
- let input_shape = [-1 as isize, config.model_dim as isize];
- let input_data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let input_placeholder =
- placeholder(&graph, &input_shape, input_data_type);
- let input_shaped_type = shaped_type(&input_shape, input_data_type);
-
- let weights_shape =
- [config.vocab_size as isize, (config.model_dim / 2) as isize];
- let weights_data_type = DataType::U8;
- let weights_shaped_type =
- shaped_type(&weights_shape, weights_data_type);
- let weights_placeholder =
- placeholder(&graph, &weights_shape, weights_data_type);
-
- let scales_shape = [config.vocab_size as isize];
- let scales_data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let scales_shaped_type =
- shaped_type(&scales_shape, scales_data_type);
- let scales_placeholder =
- placeholder(&graph, &scales_shape, scales_data_type);
-
- let weights = embeddings_dequantize_weights_subgraph(
- &graph,
- &weights_placeholder,
- &scales_placeholder,
- )
- .unwrap();
- let weights_transposed = graph.transpose(&weights, &[1, 0], None);
- let output = embeddings_readout_subgraph(
- &graph,
- &input_placeholder,
- &weights_transposed,
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
+
+ // For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim
+ let group_size = config.model_dim;
+
+ let block = QuantizedEmbeddingReadout::new_tied(
+ context,
+ data_type,
+ config.vocab_size,
+ config.model_dim,
+ group_size,
+ *embedding_quantization_mode,
+ &embeddings_tree,
)
- .unwrap();
-
- let feeds = HashMap::from([
- (&*input_placeholder, &*input_shaped_type),
- (&*weights_placeholder, &*weights_shaped_type),
- (&*scales_placeholder, &*scales_shaped_type),
- ]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(compilation_descriptor),
- );
-
- let block = MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![
- ArrayId::Main,
- ArrayId::EmbeddingsOutputWeights,
- ArrayId::EmbeddingsScales,
- ]
- .into_boxed_slice(),
- vec![ArrayId::Logits].into_boxed_slice(),
- ),
- );
+ .expect("Failed to create quantized embedding readout kernel");
Box::new(block)
},
@@ -570,8 +458,8 @@ pub fn readout_block(
data_type,
config.vocab_size,
config.model_dim,
- group_size,
- embedding_quantization_mode,
+ *group_size,
+ *embedding_quantization_mode,
&embeddings_tree,
)
.expect("Failed to create quantized embedding readout kernel");
diff --git a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
index d9724e1a4..ffa791574 100644
--- a/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/encoder_resolver.rs
@@ -1,5 +1,4 @@
-use metal::ComputeCommandEncoder;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::{CommandBufferRef, ComputeCommandEncoder};
use super::ForwardPassState;
use crate::backends::metal::encodable_block::{
@@ -7,12 +6,12 @@ use crate::backends::metal::encodable_block::{
};
pub struct EncoderResolver<'a> {
- command_buffer: &'a MPSCommandBuffer,
+ command_buffer: &'a CommandBufferRef,
encoder: Option<ComputeCommandEncoder>,
}
impl<'a> EncoderResolver<'a> {
- pub fn new(command_buffer: &'a MPSCommandBuffer) -> Self {
+ pub fn new(command_buffer: &'a CommandBufferRef) -> Self {
Self {
command_buffer,
encoder: None,
@@ -29,7 +28,6 @@ impl<'a> EncoderResolver<'a> {
if self.encoder.is_none() {
self.encoder = Some(
self.command_buffer
- .root_command_buffer()
.new_compute_command_encoder()
.to_owned(),
);
diff --git a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs b/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs
deleted file mode 100644
index e8e7322e9..000000000
--- a/crates/uzu/src/backends/metal/forward_pass/io_arrays.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-use std::{cell::RefCell, collections::HashMap};
-
-use mpsgraph::TensorData;
-use objc2::rc::Retained;
-
-use super::{ArrayId, ForwardPassState};
-use crate::backends::metal::array::MetalArray;
-
-#[derive(Clone)]
-pub struct IOArrays {
- input_arrays: Box<[ArrayId]>,
- output_arrays: Box<[ArrayId]>,
-}
-
-pub struct MPSGraphFeeds {
- pub inputs: Box<[Retained<TensorData>]>,
- pub outputs: Box<[Retained<TensorData>]>,
-}
-
-pub struct KernelFeeds {
- pub inputs: Box<[RefCell<MetalArray>]>,
- pub outputs: Box<[RefCell<MetalArray>]>,
-}
-
-impl IOArrays {
- pub fn new(
- input_arrays: Box<[ArrayId]>,
- output_arrays: Box<[ArrayId]>,
- ) -> Self {
- Self {
- input_arrays,
- output_arrays,
- }
- }
-
- fn all_ids(&self) -> Box<[ArrayId]> {
- let mut potentially_duplicated_results: Vec<ArrayId> = self
- .input_arrays
- .iter()
- .copied()
- .chain(self.output_arrays.iter().copied())
- .collect();
- potentially_duplicated_results.sort();
- potentially_duplicated_results.dedup();
- potentially_duplicated_results.into()
- }
-
- pub unsafe fn get_mpsgraph_feeds(
- &self,
- state: &mut ForwardPassState,
- ) -> MPSGraphFeeds {
- unsafe {
- let all_ids = self.all_ids();
- let all_tensor_datas =
- state.arrays(&all_ids).into_iter().map(|a_rc_refcell| {
- a_rc_refcell.borrow_mut().to_mps_tensor_data()
- });
- let id_to_result =
- HashMap::<ArrayId, Retained<TensorData>>::from_iter(
- all_ids.into_iter().zip(all_tensor_datas),
- );
- let inputs: Box<[_]> = self
- .input_arrays
- .iter()
- .map(|id| id_to_result[id].clone())
- .collect();
- let outputs: Box<[_]> = self
- .output_arrays
- .iter()
- .map(|id| id_to_result[id].clone())
- .collect();
- MPSGraphFeeds {
- inputs,
- outputs,
- }
- }
- }
-
- pub unsafe fn get_kernel_feeds(
- &self,
- state: &mut ForwardPassState,
- ) -> KernelFeeds {
- let inputs: Box<[RefCell<MetalArray>]> =
- state.arrays(&self.input_arrays);
- let outputs: Box<[RefCell<MetalArray>]> =
- state.arrays(&self.output_arrays);
-
- KernelFeeds {
- inputs,
- outputs,
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/forward_pass/mod.rs b/crates/uzu/src/backends/metal/forward_pass/mod.rs
index 37b24c7d7..72714b73b 100644
--- a/crates/uzu/src/backends/metal/forward_pass/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/mod.rs
@@ -1,9 +1,7 @@
pub mod cache_layers;
mod encoder_resolver;
-mod io_arrays;
pub mod kv_cache_layer;
mod model_shape;
-mod mpsgraph_block;
mod scratch_buffers;
pub mod short_conv_layer;
pub mod ssm_layer;
@@ -13,12 +11,10 @@ pub mod traces;
pub use cache_layers::{CacheLayer, CacheLayers};
pub use encoder_resolver::EncoderResolver;
-pub use io_arrays::IOArrays;
pub use kv_cache_layer::{
AttentionBiasUpdate, INVALID_POSITION, KVCacheLayer, KVCacheLayerState,
};
pub use model_shape::ModelShape;
-pub use mpsgraph_block::MPSGraphBlock;
pub use scratch_buffers::ScratchBuffers;
pub use short_conv_layer::ShortConvLayer;
pub use ssm_layer::SSMLayer;
diff --git a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs b/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
deleted file mode 100644
index f285c3f6e..000000000
--- a/crates/uzu/src/backends/metal/forward_pass/mpsgraph_block.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-use mpsgraph::{
- CommandBuffer as MPSCommandBuffer, Executable,
- ExecutableExecutionDescriptor, TensorData,
-};
-use objc2::rc::Retained;
-
-use super::{
- EncodableBlock, EncodingParameters, ForwardPassState, io_arrays::IOArrays,
-};
-
-pub struct MPSGraphBlock {
- executable: Retained<Executable>,
- execution_descriptor: Retained<ExecutableExecutionDescriptor>,
- arguments: IOArrays,
-}
-
-impl MPSGraphBlock {
- pub fn new(
- executable: Retained<Executable>,
- execution_descriptor: Retained<ExecutableExecutionDescriptor>,
- arguments: IOArrays,
- ) -> Self {
- Self {
- executable,
- execution_descriptor,
- arguments,
- }
- }
-}
-
-impl EncodableBlock for MPSGraphBlock {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- unsafe {
- let feeds = self.arguments.get_mpsgraph_feeds(state);
-
- let inputs_refs: Vec<&TensorData> =
- feeds.inputs.iter().map(|retained_td| &**retained_td).collect();
- let inputs_slice: &[&TensorData] = &inputs_refs;
-
- let outputs_refs: Vec<&TensorData> = feeds
- .outputs
- .iter()
- .map(|retained_td| &**retained_td)
- .collect();
- let outputs_slice: &[&TensorData] = &outputs_refs;
-
- let maybe_outputs_slice = if outputs_slice.is_empty() {
- None
- } else {
- Some(outputs_slice)
- };
-
- let execution_descriptor = self.execution_descriptor.clone();
- execution_descriptor
- .set_enable_commit_and_continue(parameters.enable_commit);
-
- let root_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let _ = self.executable.encode_to_command_buffer(
- command_buffer,
- inputs_slice,
- maybe_outputs_slice,
- Some(&execution_descriptor),
- );
- if parameters.wait_until_completed {
- command_buffer.commit_and_continue();
- root_command_buffer.wait_until_completed();
- }
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
index 86654b28f..aabb8a89f 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -1020,7 +1020,7 @@ impl ForwardPassState {
pub fn encode_copy_array(
&self,
- command_buffer: &mpsgraph::CommandBuffer,
+ command_buffer: &metal::CommandBufferRef,
source_array_id: ArrayId,
destination_array: RefCell<MetalArray>,
) {
@@ -1037,9 +1037,7 @@ impl ForwardPassState {
src_borrow.size_in_bytes()
);
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let blit_encoder = mtl_command_buffer.new_blit_command_encoder();
+ let blit_encoder = command_buffer.new_blit_command_encoder();
blit_encoder.copy_from_buffer(
&src_buf,
0,
diff --git a/crates/uzu/src/backends/metal/graph/common.rs b/crates/uzu/src/backends/metal/graph/common.rs
deleted file mode 100644
index 078c10c68..000000000
--- a/crates/uzu/src/backends/metal/graph/common.rs
+++ /dev/null
@@ -1,257 +0,0 @@
-use std::rc::Rc;
-
-use mpsgraph::{Graph, ShapedType, Tensor};
-use objc2::rc::Retained;
-use thiserror::Error;
-
-use crate::{
- Array, DataType,
- backends::metal::{MTLContext, MetalArray},
- config::Activation,
- parameters::{ParameterLoaderError, ParameterTree},
-};
-
-pub fn data_type_of(tensor: &Tensor) -> DataType {
- DataType::from(tensor.data_type())
-}
-
-#[derive(Debug, Error)]
-pub enum GraphConstructionError {
- #[error("Failed to load weights: {0}")]
- ParameterLoaderError(#[from] ParameterLoaderError),
- #[error(
- "Incompatible shapes. Node {node_name} at subtree {node_path:?} expected to have shape {expected:?} but got {actual:?}"
- )]
- IncompatibleShapes {
- node_path: Option<String>,
- node_name: String,
- expected: Box<[usize]>,
- actual: Box<[usize]>,
- },
- #[error(
- "Incompatible data types. Node {node_name} at subtree {node_path:?} expected to have data type {expected:?} but got {actual:?}"
- )]
- IncompatibleDataTypes {
- node_path: Option<String>,
- node_name: String,
- expected: DataType,
- actual: DataType,
- },
- #[error("Failed to run callable")]
- CallableResultError {
- name: String,
- },
-}
-
-pub fn activation(
- graph: &Graph,
- config: &Activation,
- input: &Tensor,
- data_type: DataType,
-) -> Retained<Tensor> {
- match config {
- Activation::SiLU {
- ..
- } => {
- let sigmoid_result = graph.sigmoid(input, Some("sigmoid"));
- graph.multiplication(input, &sigmoid_result, Some("silu_output"))
- },
- Activation::Gelu => gelu(graph, input, data_type, true),
- Activation::Identity => graph.addition(
- input,
- &graph.constant_with_scalar(0.0_f64, None, data_type.into()),
- Some("identity_output"),
- ),
- }
-}
-
-pub fn gelu(
- graph: &Graph,
- input: &Tensor,
- data_type: DataType,
- approximate: bool,
-) -> Retained<Tensor> {
- if approximate {
- let input_power_2 = graph.multiplication(input, input, None);
- let input_power_3 = graph.multiplication(&input_power_2, input, None);
- let input_power_3_multiply_const = graph.multiplication(
- &graph.constant_with_scalar(
- 0.044715 as f64,
- None,
- data_type.into(),
- ),
- &input_power_3,
- None,
- );
- let input_add_input_power_3_multiply_const =
- graph.addition(input, &input_power_3_multiply_const, None);
- let tanh = graph.tanh(
- &graph.multiplication(
- &graph.constant_with_scalar(
- (2.0 / std::f64::consts::PI).sqrt(),
- None,
- data_type.into(),
- ),
- &input_add_input_power_3_multiply_const,
- None,
- ),
- None,
- );
- let tanh_add_one = graph.addition(
- &graph.constant_with_scalar(1.0 as f64, None, data_type.into()),
- &tanh,
- None,
- );
- let input_multiply_tanh_add_one =
- graph.multiplication(input, &tanh_add_one, None);
- let result = graph.multiplication(
- &input_multiply_tanh_add_one,
- &graph.constant_with_scalar(0.5 as f64, None, data_type.into()),
- None,
- );
- result
- } else {
- let input_negative = graph.multiplication(
- &graph.constant_with_scalar(-1.0 as f64, None, data_type.into()),
- input,
- None,
- );
- let input_negative_divide_sqrt_2 = graph.division(
- &input_negative,
- &graph.constant_with_scalar(
- (2.0 as f64).sqrt(),
- None,
- data_type.into(),
- ),
- None,
- );
- let erf = graph.erf(&input_negative_divide_sqrt_2, None);
- let erf_negative = graph.multiplication(
- &graph.constant_with_scalar(-1.0 as f64, None, data_type.into()),
- &erf,
- None,
- );
- let erfc = graph.addition(
- &graph.constant_with_scalar(1.0 as f64, None, data_type.into()),
- &erf_negative,
- None,
- );
- let input_divide_2 = graph.division(
- &input,
- &graph.constant_with_scalar(2.0 as f64, None, data_type.into()),
- None,
- );
- let result = graph.multiplication(&input_divide_2, &erfc, None);
- result
- }
-}
-
-pub fn placeholder(
- graph: &Graph,
- shape: &[isize],
- data_type: DataType,
-) -> Retained<Tensor> {
- graph.placeholder(Some(shape), data_type.into(), Some("input"))
-}
-
-pub fn shaped_type(
- shape: &[isize],
- data_type: DataType,
-) -> Retained<ShapedType> {
- ShapedType::new_with_shape_data_type(Some(shape), data_type.into())
-}
-
-fn map_last_dimension<F: FnMut(usize) -> usize>(
- mut f: F,
- shape: &[usize],
-) -> Box<[usize]> {
- shape
- .iter()
- .enumerate()
- .map(|(i, dim)| {
- if i == shape.len() - 1 {
- f(*dim)
- } else {
- *dim
- }
- })
- .collect()
-}
-
-fn unpacked_shape_from_packed_shape(packed_shape: &[usize]) -> Box<[usize]> {
- map_last_dimension(|dim| dim * 2, packed_shape)
-}
-
-fn packed_shape_from_unpacked_shape(packed_shape: &[usize]) -> Box<[usize]> {
- map_last_dimension(|dim| dim / 2, packed_shape)
-}
-
-pub fn i4_constant_from_packed_u8_array(
- graph: &Graph,
- data: &MetalArray,
-) -> Retained<Tensor> {
- assert!(data.data_type() == DataType::U8);
- graph.constant_with_data(
- data.buffer(),
- &unpacked_shape_from_packed_shape(data.shape()),
- DataType::I4.into(),
- )
-}
-
-pub fn load_constant(
- graph: &Graph,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- name: &str,
- expected_shape: &[usize],
- expected_data_type: DataType,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let parameter = parameter_tree.leaf(name)?;
- let result = if expected_data_type == DataType::U4 {
- if parameter.data_type() != DataType::U8 {
- return Err(GraphConstructionError::IncompatibleDataTypes {
- node_path: parameter_tree.path_prefix().map(str::to_string),
- node_name: name.to_string(),
- expected: DataType::U8,
- actual: parameter.data_type(),
- });
- };
- let packed_expected_shape =
- packed_shape_from_unpacked_shape(expected_shape);
- if parameter.shape() != &*packed_expected_shape {
- return Err(GraphConstructionError::IncompatibleShapes {
- node_path: parameter_tree.path_prefix().map(str::to_string),
- node_name: name.to_string(),
- expected: packed_expected_shape.into(),
- actual: parameter.shape().into(),
- });
- };
- graph.constant_with_data(
- parameter.buffer(),
- expected_shape,
- DataType::U4.into(),
- )
- } else {
- if parameter.shape() != expected_shape {
- return Err(GraphConstructionError::IncompatibleShapes {
- node_path: parameter_tree.path_prefix().map(str::to_string),
- node_name: name.to_string(),
- expected: expected_shape.into(),
- actual: parameter.shape().into(),
- });
- }
- if parameter.data_type() != expected_data_type {
- return Err(GraphConstructionError::IncompatibleDataTypes {
- node_path: parameter_tree.path_prefix().map(str::to_string),
- node_name: name.to_string(),
- expected: expected_data_type,
- actual: parameter.data_type(),
- });
- };
- graph.constant_with_data(
- parameter.buffer(),
- parameter.shape(),
- parameter.data_type().into(),
- )
- };
- Ok(result)
-}
diff --git a/crates/uzu/src/backends/metal/graph/embedding.rs b/crates/uzu/src/backends/metal/graph/embedding.rs
deleted file mode 100644
index e712a35dd..000000000
--- a/crates/uzu/src/backends/metal/graph/embedding.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-use mpsgraph::{DequantizationArguments, Graph, Tensor};
-use objc2::rc::Retained;
-
-use super::GraphConstructionError;
-use crate::config::EmbeddingConfig;
-
-pub fn embeddings_dequantize_weights_subgraph(
- graph: &Retained<Graph>,
- weights: &Retained<Tensor>,
- scales: &Retained<Tensor>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let result = graph.dequantize(
- &weights,
- DequantizationArguments::ScaleTensorZeroPointDataTypeAxis {
- scale_tensor: &scales,
- zero_point: 0.0,
- data_type: scales.data_type(),
- axis: 0,
- },
- None,
- );
- Ok(result)
-}
-
-pub fn embeddings_embed_subgraph(
- graph: &Retained<Graph>,
- config: &EmbeddingConfig,
- input: &Retained<Tensor>,
- weights: &Retained<Tensor>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let result = graph.gather_with_updates(&weights, input, 0, 0, None);
- if let Some(scale) = config.common().input_scale {
- let scale_tensor =
- graph.constant_with_scalar(scale as f64, None, result.data_type());
- let scaled_result = graph.multiplication(&result, &scale_tensor, None);
- Ok(scaled_result)
- } else {
- Ok(result)
- }
-}
-
-pub fn embeddings_readout_subgraph(
- graph: &Retained<Graph>,
- input: &Retained<Tensor>,
- weights: &Retained<Tensor>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let result = graph.matrix_multiplication(input, weights, None);
- Ok(result)
-}
diff --git a/crates/uzu/src/backends/metal/graph/linear.rs b/crates/uzu/src/backends/metal/graph/linear.rs
deleted file mode 100644
index 1422d5b49..000000000
--- a/crates/uzu/src/backends/metal/graph/linear.rs
+++ /dev/null
@@ -1,398 +0,0 @@
-use std::rc::Rc;
-
-use mpsgraph::{DequantizationArguments, Graph, Tensor};
-use objc2::rc::Retained;
-
-use super::{super::MTLContext, GraphConstructionError, load_constant};
-use crate::{
- DataType,
- config::{ConfigDataType, LinearConfig, QuantizationConfig},
- parameters::ParameterTree,
-};
-
-fn full_precision_weights_subgraph<const N: usize>(
- graph: &Graph,
- precision: DataType,
- input_dim: usize,
- output_dims: [usize; N],
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let output_dim_sum: usize = output_dims.iter().sum();
- load_constant(
- graph,
- parameter_tree,
- "weights",
- &[output_dim_sum, input_dim],
- precision,
- )
-}
-
-fn quantized_weights_subgraph<const N: usize>(
- graph: &Graph,
- config: &QuantizationConfig,
- input_dim: usize,
- output_dims: [usize; N],
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let output_dim_sum: usize = output_dims.iter().sum();
-
- let weights = load_constant(
- graph,
- parameter_tree,
- "weights",
- &[input_dim, output_dim_sum],
- config.weight_quantization_mode.into(),
- )?;
-
- let scales = load_constant(
- graph,
- parameter_tree,
- "scales",
- &[input_dim, output_dim_sum / config.group_size],
- config.activation_precision.into(),
- )?;
-
- let zero_points = load_constant(
- graph,
- parameter_tree,
- "zero_points",
- &[input_dim, output_dim_sum / config.group_size],
- config.weight_quantization_mode.into(),
- )?;
-
- let result = graph.dequantize(
- &weights,
- DequantizationArguments::ScaleTensorZeroPointTensorDataType {
- scale_tensor: &scales,
- zero_point_tensor: &zero_points,
- data_type: scales.data_type(),
- },
- None,
- );
- Ok(result)
-}
-
-fn mlx_quantized_weights_subgraph<const N: usize>(
- graph: &Graph,
- config: &QuantizationConfig,
- input_dim: usize,
- output_dims: [usize; N],
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let output_dim_sum: usize = output_dims.iter().sum();
-
- let weights = load_constant(
- graph,
- parameter_tree,
- "weights",
- &[input_dim, output_dim_sum],
- config.weight_quantization_mode.into(),
- )?;
-
- let scales = load_constant(
- graph,
- parameter_tree,
- "scales",
- &[input_dim, output_dim_sum / config.group_size],
- config.activation_precision.into(),
- )?;
-
- let deq_biases = load_constant(
- graph,
- parameter_tree,
- "deq_biases",
- &[input_dim, output_dim_sum / config.group_size],
- config.activation_precision.into(),
- )?;
-
- let result = graph.dequantize(
- &weights,
- DequantizationArguments::ScaleTensorZeroPointTensorDataType {
- scale_tensor: &scales,
- zero_point_tensor: &deq_biases,
- data_type: scales.data_type(),
- },
- None,
- );
- Ok(result)
-}
-
-fn biases_subgraph<const N: usize>(
- graph: &Graph,
- precision: DataType,
- output_dims: [usize; N],
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let output_dim_sum: usize = output_dims.iter().sum();
- load_constant(graph, parameter_tree, "biases", &[output_dim_sum], precision)
-}
-
-struct LoraWeights {
- down_weights: Retained<Tensor>,
- up_weights: Box<[Retained<Tensor>]>,
-}
-
-fn lora_weights_subgraph<const N: usize>(
- graph: &Graph,
- precision: DataType,
- input_dim: usize,
- output_dims: [usize; N],
- lora_rank: usize,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<LoraWeights, GraphConstructionError> {
- let down_weights = load_constant(
- graph,
- parameter_tree,
- "down_weights",
- &[input_dim, lora_rank * N],
- precision,
- )?;
- let up_weights = output_dims
- .iter()
- .enumerate()
- .map(|(i, output_dim)| {
- load_constant(
- graph,
- parameter_tree,
- &format!("up_weights.{}", i),
- &[lora_rank, *output_dim],
- precision,
- )
- })
- .collect::<Result<Box<[_]>, _>>()?;
- Ok(LoraWeights {
- down_weights,
- up_weights: up_weights,
- })
-}
-
-fn full_precision_matmul_subgraph<const N: usize>(
- graph: &Graph,
- precision: DataType,
- input_dim: usize,
- output_dims: [usize; N],
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let weights = full_precision_weights_subgraph::<N>(
- graph,
- precision,
- input_dim,
- output_dims,
- parameter_tree,
- )?;
- Ok(graph.transpose(
- &graph.matrix_multiplication(
- &weights,
- &graph.transpose(input, &[1, 0], None),
- None,
- ),
- &[1, 0],
- None,
- ))
-}
-
-fn quantized_matmul_subgraph<const N: usize>(
- graph: &Graph,
- config: &QuantizationConfig,
- input_dim: usize,
- output_dims: [usize; N],
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let weights = quantized_weights_subgraph::<N>(
- graph,
- config,
- input_dim,
- output_dims,
- parameter_tree,
- )?;
- Ok(graph.matrix_multiplication(input, &weights, None))
-}
-
-fn mlx_quantized_matmul_subgraph<const N: usize>(
- graph: &Graph,
- config: &QuantizationConfig,
- input_dim: usize,
- output_dims: [usize; N],
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let weights = mlx_quantized_weights_subgraph::<N>(
- graph,
- config,
- input_dim,
- output_dims,
- parameter_tree,
- )?;
- Ok(graph.matrix_multiplication(input, &weights, None))
-}
-
-fn lora_subgraph<const N: usize>(
- graph: &Graph,
- precision: DataType,
- lora_rank: usize,
- input_dim: usize,
- output_dims: [usize; N],
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let lora_weights = lora_weights_subgraph(
- graph,
- precision,
- input_dim,
- output_dims,
- lora_rank,
- parameter_tree,
- )?;
-
- let lora_inner = graph.transpose(
- &graph.matrix_multiplication(
- &graph.transpose(&lora_weights.down_weights, &[1, 0], None),
- &graph.transpose(input, &[1, 0], None),
- None,
- ),
- &[1, 0],
- None,
- );
-
- let lora_inner_splits =
- graph.split_num_splits(&lora_inner, output_dims.len() as u64, 1, None);
- assert_eq!(lora_inner_splits.len(), lora_weights.up_weights.len());
-
- let lora_outputs = lora_inner_splits
- .iter()
- .zip(lora_weights.up_weights.iter())
- .map(|(lora_inner_split, up_weights)| {
- graph.transpose(
- &graph.matrix_multiplication(
- &graph.transpose(&up_weights, &[1, 0], None),
- &graph.transpose(&lora_inner_split, &[1, 0], None),
- None,
- ),
- &[1, 0],
- None,
- )
- })
- .collect::<Box<[Retained<Tensor>]>>();
-
- Ok(graph.concat_tensors(
- &lora_outputs
- .iter()
- .map(|tensor| &**tensor)
- .collect::<Box<[&Tensor]>>(),
- 1,
- false,
- None,
- ))
-}
-
-fn qlora_matmul_subgraph<const N: usize>(
- graph: &Graph,
- lora_rank: usize,
- lora_scale: f32,
- config: &QuantizationConfig,
- input_dim: usize,
- output_dims: [usize; N],
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let quantized_matmul_result = quantized_matmul_subgraph::<N>(
- graph,
- config,
- input_dim,
- output_dims,
- input,
- parameter_tree,
- )?;
- let lora_result = lora_subgraph::<N>(
- graph,
- config.activation_precision.into(),
- lora_rank,
- input_dim,
- output_dims,
- input,
- parameter_tree,
- )?;
- let scaled_lora_result = graph.multiplication(
- &lora_result,
- &graph.constant_with_scalar(
- lora_scale as f64,
- None,
- <ConfigDataType as Into<DataType>>::into(
- config.activation_precision,
- )
- .into(),
- ),
- None,
- );
- Ok(graph.addition(&quantized_matmul_result, &scaled_lora_result, None))
-}
-
-pub fn linear_subgraph<const N: usize>(
- graph: &Graph,
- config: &LinearConfig,
- input_dim: usize,
- output_dims: [usize; N],
- has_biases: bool,
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let matmul_result = match config {
- LinearConfig::FullPrecision {
- precision,
- } => full_precision_matmul_subgraph(
- graph,
- (*precision).into(),
- input_dim,
- output_dims,
- input,
- parameter_tree,
- )?,
- LinearConfig::Quantized(quantization_config) => {
- quantized_matmul_subgraph(
- graph,
- quantization_config,
- input_dim,
- output_dims,
- input,
- parameter_tree,
- )?
- },
- LinearConfig::MLXQuantized(quantization_config) => {
- mlx_quantized_matmul_subgraph(
- graph,
- quantization_config,
- input_dim,
- output_dims,
- input,
- parameter_tree,
- )?
- },
- LinearConfig::QLoRA {
- quantization: quantization_config,
- lora_rank,
- lora_scale,
- } => qlora_matmul_subgraph(
- graph,
- *lora_rank,
- *lora_scale,
- quantization_config,
- input_dim,
- output_dims,
- input,
- parameter_tree,
- )?,
- };
- if has_biases {
- let biases = biases_subgraph(
- graph,
- config.activation_precision().into(),
- output_dims,
- parameter_tree,
- )?;
- Ok(graph.addition(&matmul_result, &biases, None))
- } else {
- Ok(matmul_result)
- }
-}
diff --git a/crates/uzu/src/backends/metal/graph/mlp.rs b/crates/uzu/src/backends/metal/graph/mlp.rs
deleted file mode 100644
index 509e30dc3..000000000
--- a/crates/uzu/src/backends/metal/graph/mlp.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-use std::rc::Rc;
-
-use mpsgraph::{Graph, Tensor};
-use objc2::rc::Retained;
-
-use super::{
- super::MTLContext, GraphConstructionError, common::activation,
- linear::linear_subgraph,
-};
-use crate::{config::MLPConfig, parameters::ParameterTree};
-
-pub fn mlp_subgraph(
- graph: &Graph,
- config: &MLPConfig,
- model_dim: usize,
- hidden_dim: usize,
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- match config {
- MLPConfig::Dense(dense) => {
- let up_tree = parameter_tree.subtree("up_projection")?;
- let down_tree = parameter_tree.subtree("down_projection")?;
-
- let fused_hidden_gate = linear_subgraph(
- graph,
- &dense.linear_config,
- model_dim,
- [hidden_dim, hidden_dim],
- false,
- input,
- &up_tree,
- )?;
-
- let split_results =
- graph.split_num_splits(&fused_hidden_gate, 2, 1, None);
- let up_proj = &split_results[0];
- let gate = &split_results[1];
-
- let activated_gate = activation(
- graph,
- &dense.activation,
- &*gate,
- dense.linear_config.activation_precision().into(),
- );
- let hidden = graph.multiplication(&activated_gate, up_proj, None);
-
- let result = linear_subgraph(
- graph,
- &dense.linear_config,
- hidden_dim,
- [model_dim],
- false,
- &*hidden,
- &down_tree,
- )?;
-
- Ok(result)
- },
- MLPConfig::MixtureOfExperts(_moe) => {
- unreachable!("MoE uses MoeBlockEncodable, not MPSGraph")
- },
- }
-}
diff --git a/crates/uzu/src/backends/metal/graph/mod.rs b/crates/uzu/src/backends/metal/graph/mod.rs
deleted file mode 100644
index ddb1f870f..000000000
--- a/crates/uzu/src/backends/metal/graph/mod.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-pub mod common;
-use common::{GraphConstructionError, load_constant};
-pub use common::{placeholder, shaped_type};
-mod linear;
-pub use linear::linear_subgraph;
-mod mlp;
-pub use mlp::mlp_subgraph;
-mod embedding;
-pub use embedding::{
- embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
- embeddings_readout_subgraph,
-};
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
new file mode 100644
index 000000000..fb5e88d00
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
@@ -0,0 +1,71 @@
+#include <metal_stdlib>
+using namespace metal;
+
+enum ActivationType : ushort {
+ ACT_SILU = 0,
+ ACT_GELU = 1,
+};
+
+template <typename T>
+inline T silu(T x) {
+ float xf = float(x);
+ float yf = xf / (1.0f + exp(-xf));
+ return T(yf);
+}
+
+template <typename T>
+inline T gelu_approx(T x) {
+ constexpr float k0 = 0.044715f;
+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
+ float xf = float(x);
+ float yf = 0.5f * xf * (1.0f + tanh(k1 * (xf + k0 * xf * xf * xf)));
+ return T(yf);
+}
+
+template <typename T>
+inline T activate(T x, ushort act) {
+ if (act == ACT_SILU)
+ return silu(x);
+ return gelu_approx(x);
+}
+
+template <typename T>
+[[kernel]] void activation(
+ const device T* input [[buffer(0)]],
+ device T* output [[buffer(1)]],
+ const constant int& N [[buffer(2)]],
+ const constant ushort& act_type [[buffer(3)]],
+ uint tid [[thread_position_in_grid]]
+) {
+ if (int(tid) >= N)
+ return;
+ output[tid] = activate(input[tid], act_type);
+}
+
+// Explicit instantiations
+template [[host_name("activation_f16")]] [[kernel]] void
+activation<half>(
+ const device half* input [[buffer(0)]],
+ device half* output [[buffer(1)]],
+ const constant int& N [[buffer(2)]],
+ const constant ushort& act_type [[buffer(3)]],
+ uint tid [[thread_position_in_grid]]
+);
+
+template [[host_name("activation_f32")]] [[kernel]] void
+activation<float>(
+ const device float* input [[buffer(0)]],
+ device float* output [[buffer(1)]],
+ const constant int& N [[buffer(2)]],
+ const constant ushort& act_type [[buffer(3)]],
+ uint tid [[thread_position_in_grid]]
+);
+
+template [[host_name("activation_bf16")]] [[kernel]] void
+activation<bfloat>(
+ const device bfloat* input [[buffer(0)]],
+ device bfloat* output [[buffer(1)]],
+ const constant int& N [[buffer(2)]],
+ const constant ushort& act_type [[buffer(3)]],
+ uint tid [[thread_position_in_grid]]
+);
diff --git a/crates/uzu/src/backends/metal/kernel/activation/mod.rs b/crates/uzu/src/backends/metal/kernel/activation/mod.rs
new file mode 100644
index 000000000..c191d30d6
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/activation/mod.rs
@@ -0,0 +1,90 @@
+use std::mem::size_of;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+ config::Activation,
+};
+
+pub struct ActivationKernel {
+ pipeline: MTLComputePipelineState,
+}
+
+impl ActivationKernel {
+ fn kernel_name_for_type(
+ data_type: DataType
+ ) -> Result<&'static str, MTLError> {
+ match data_type {
+ DataType::F16 => Ok("activation_f16"),
+ DataType::F32 => Ok("activation_f32"),
+ DataType::BF16 => Ok("activation_bf16"),
+ other => Err(MTLError::Generic(format!(
+ "Unsupported dtype for activation: {:?}",
+ other
+ ))),
+ }
+ }
+
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ ) -> Result<Self, MTLError> {
+ let fn_name = Self::kernel_name_for_type(data_type)?;
+ let (pipeline, _) =
+ context.compute_pipeline_state_with_reflection(fn_name, None)?;
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ fn act_code(act: &Activation) -> u16 {
+ match act {
+ Activation::SiLU {
+ ..
+ } => 0,
+ Activation::Gelu => 1,
+ Activation::Identity => {
+ panic!("Identity activation is not supported for kernel")
+ },
+ }
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ activation: &Activation,
+ input_buffer: &MTLBuffer,
+ output_buffer: &MTLBuffer,
+ n: usize,
+ ) -> Result<(), MTLError> {
+ let act_code = Self::act_code(activation);
+ let threads_per_tg = 256u64;
+ let threadgroups = (n as u64 + threads_per_tg - 1) / threads_per_tg;
+
+ encoder.set_compute_pipeline_state(&self.pipeline);
+ encoder.set_buffer(0, Some(input_buffer), 0);
+ encoder.set_buffer(1, Some(output_buffer), 0);
+ encoder.set_bytes(
+ 2,
+ size_of::<i32>() as u64,
+ &(n as i32) as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 3,
+ size_of::<u16>() as u64,
+ &act_code as *const u16 as *const _,
+ );
+
+ encoder.dispatch_thread_groups(
+ MTLSize::new(threadgroups, 1, 1),
+ MTLSize::new(threads_per_tg, 1, 1),
+ );
+ Ok(())
+ }
+}
+
diff --git a/crates/uzu/src/backends/metal/kernel/data_type.rs b/crates/uzu/src/backends/metal/kernel/data_type.rs
index 1e28cf2e8..1237a4185 100644
--- a/crates/uzu/src/backends/metal/kernel/data_type.rs
+++ b/crates/uzu/src/backends/metal/kernel/data_type.rs
@@ -17,9 +17,9 @@ impl KernelDataType {
}
}
-impl Into<DataType> for KernelDataType {
- fn into(self) -> DataType {
- match self {
+impl From<KernelDataType> for DataType {
+ fn from(val: KernelDataType) -> Self {
+ match val {
KernelDataType::BFloat16 => DataType::BF16,
KernelDataType::Float16 => DataType::F16,
KernelDataType::Float32 => DataType::F32,
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
index 647eff7f4..16f9c1ae9 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
@@ -7,7 +7,7 @@ use metal::{
use super::super::MTLContext;
use crate::{
- DataType, backends::metal::encodable_block::QuantizedEmbeddingError,
+ DataType, backends::metal::encodable_block::EmbeddingError,
config::QuantizationMode,
};
@@ -32,13 +32,13 @@ impl FullPrecisionEmbeddingLookupKernel {
pub fn new(
mtl_context: &MTLContext,
data_type: DataType,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
let dtype_suffix = match data_type {
DataType::F32 => "f32",
DataType::F16 => "f16",
DataType::BF16 => "bf16",
other => {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ return Err(EmbeddingError::UnsupportedDataType(
other,
));
},
@@ -48,7 +48,7 @@ impl FullPrecisionEmbeddingLookupKernel {
let (pipeline, _) = mtl_context
.compute_pipeline_state_with_reflection(&kernel_name, None)
- .map_err(QuantizedEmbeddingError::MetalError)?;
+ .map_err(EmbeddingError::MetalError)?;
Ok(Self {
pipeline,
@@ -59,7 +59,7 @@ impl FullPrecisionEmbeddingLookupKernel {
&self,
encoder: &ComputeCommandEncoderRef,
args: FullPrecisionEmbeddingLookupArguments,
- ) -> Result<(), QuantizedEmbeddingError> {
+ ) -> Result<(), EmbeddingError> {
encoder.set_compute_pipeline_state(&self.pipeline);
encoder.set_buffer(0, Some(args.token_ids_buffer), 0);
@@ -126,13 +126,13 @@ impl QuantizedEmbeddingLookupKernel {
mtl_context: &MTLContext,
data_type: DataType,
mode: QuantizationMode,
- ) -> Result<Self, QuantizedEmbeddingError> {
+ ) -> Result<Self, EmbeddingError> {
let dtype_suffix = match data_type {
DataType::F32 => "f32",
DataType::F16 => "f16",
DataType::BF16 => "bf16",
other => {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ return Err(EmbeddingError::UnsupportedDataType(
other,
));
},
@@ -149,7 +149,7 @@ impl QuantizedEmbeddingLookupKernel {
let (pipeline, _) = mtl_context
.compute_pipeline_state_with_reflection(&kernel_name, None)
- .map_err(QuantizedEmbeddingError::MetalError)?;
+ .map_err(EmbeddingError::MetalError)?;
Ok(Self {
pipeline,
@@ -160,7 +160,7 @@ impl QuantizedEmbeddingLookupKernel {
&self,
encoder: &ComputeCommandEncoderRef,
args: QuantizedEmbeddingLookupArguments,
- ) -> Result<(), QuantizedEmbeddingError> {
+ ) -> Result<(), EmbeddingError> {
encoder.set_compute_pipeline_state(&self.pipeline);
// Set buffers
diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
index bae2543ef..5426957ab 100644
--- a/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/mod.rs
@@ -1,11 +1,10 @@
use std::mem;
use metal::{
- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
+ Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer, CommandBufferRef,
ComputeCommandEncoder, ComputePipelineState, MTLSize,
foreign_types::{ForeignType, ForeignTypeRef},
};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
use crate::backends::metal::{
MTLContext,
@@ -86,7 +85,7 @@ impl EncodableBlock for ScalePadNormalizeImage {
fn encode(
&self,
_state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &CommandBufferRef,
_parameters: &EncodingParameters,
) {
let _ = command_buffer;
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 0528c0967..e21378a11 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,10 +1,11 @@
+pub mod activation;
pub mod attention;
mod data_type;
pub mod embedding;
pub mod kv_cache_update;
pub mod layer_norm;
-pub mod matmul;
pub mod mask_update;
+pub mod matmul;
pub mod media_kernels;
pub mod mlp;
pub mod moe;
@@ -21,6 +22,7 @@ mod tensor_add_swap;
mod tensor_copy;
pub mod token_copy;
+pub use activation::ActivationKernel;
pub use attention::{
AttentionError, AttentionKernel, AttentionKernelVariant,
AttentionSinglePassArguments, AttentionTwoPassArguments,
@@ -29,8 +31,8 @@ pub use attention::{
pub use data_type::KernelDataType;
pub use kv_cache_update::KVCacheUpdate;
pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};
-pub use matmul::{MatmulArguments, MatmulKernel};
pub use mask_update::MaskUpdateKernel;
+pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,
MoeCountsOffsetsFusedError, MoeCountsOffsetsFusedKernel,
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs
index fd6b0afd4..f42a4da4d 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/mod.rs
@@ -1,8 +1,8 @@
use std::mem::size_of;
use metal::{
- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
+ Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState,
};
use super::{
@@ -37,7 +37,7 @@ impl TensorAddBias {
output: &MTLBuffer,
num_cols: usize,
total_len: usize,
- command_buffer: &MTLCommandBuffer,
+ command_buffer: &CommandBufferRef,
predicate: Option<&MTLBuffer>,
) {
let encoder = command_buffer.new_compute_command_encoder();
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
index 1dae0f2c3..d51749746 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/mod.rs
@@ -1,7 +1,7 @@
use std::mem::size_of;
use metal::{
- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
+ Buffer as MTLBuffer, CommandBufferRef,
ComputePipelineState as MTLComputePipelineState,
};
@@ -36,7 +36,7 @@ impl TensorAddSwapKernel {
skip_buffer: &MTLBuffer,
main_buffer: &MTLBuffer,
length: usize,
- command_buffer: &MTLCommandBuffer,
+ command_buffer: &CommandBufferRef,
) {
let compute_encoder = command_buffer.new_compute_command_encoder();
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
index 743ff1d23..7d66ea8a3 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/mod.rs
@@ -1,7 +1,7 @@
use std::mem::size_of;
use metal::{
- Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
+ Buffer as MTLBuffer, CommandBufferRef,
ComputePipelineState as MTLComputePipelineState,
};
@@ -35,7 +35,7 @@ impl TensorCopyKernel {
source_buffer: &MTLBuffer,
destination_buffer: &MTLBuffer,
length: usize,
- command_buffer: &MTLCommandBuffer,
+ command_buffer: &CommandBufferRef,
) {
let compute_encoder = command_buffer.new_compute_command_encoder();
compute_encoder.set_label("Tensor Copy");
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
index 3b85afdc4..74d5feee3 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -6,9 +6,7 @@ use std::{
rc::Rc,
};
-use metal::Event as MTLEvent;
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use objc2::rc::Retained;
+use metal::{CommandBuffer, Event as MTLEvent};
use super::{
CacheLayers, Decoder, KVCacheUpdate, KernelDataType, MTLContext,
@@ -131,7 +129,7 @@ impl AsyncBuffers {
pub struct LanguageModelGeneratorContext {
pub mtl_context: Rc<MTLContext>,
- pub command_buffer: Retained<MPSCommandBuffer>,
+ pub command_buffer: CommandBuffer,
pub cache_layers: Rc<RefCell<CacheLayers>>,
pub shared_buffers: Rc<RefCell<SharedBuffers>>,
@@ -163,8 +161,7 @@ impl LanguageModelGeneratorContext {
let mtl_command_queue =
mtl_device.new_command_queue_with_max_command_buffer_count(1024);
- let command_buffer =
- MPSCommandBuffer::from_command_queue(&mtl_command_queue);
+ let command_buffer = mtl_command_queue.new_command_buffer().to_owned();
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -308,10 +305,7 @@ impl LanguageModelGeneratorContext {
}
pub fn reset_command_buffer(&mut self) {
- objc2::rc::autoreleasepool(|_pool| {
- self.command_buffer = MPSCommandBuffer::from_command_queue(
- &self.mtl_context.command_queue,
- );
- });
+ self.command_buffer =
+ self.mtl_context.command_queue.new_command_buffer().to_owned();
}
}
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 908ddbb34..c55780fed 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -6,7 +6,6 @@ mod context;
pub mod encodable_block;
pub mod error;
pub mod forward_pass;
-pub mod graph;
pub mod image;
pub mod kernel;
mod language_model_generator_context;
diff --git a/crates/uzu/src/config/common.rs b/crates/uzu/src/config/common.rs
index f99608cae..92bf13805 100644
--- a/crates/uzu/src/config/common.rs
+++ b/crates/uzu/src/config/common.rs
@@ -13,9 +13,9 @@ pub enum ConfigDataType {
Int8,
}
-impl Into<DataType> for ConfigDataType {
- fn into(self) -> DataType {
- match self {
+impl From<ConfigDataType> for DataType {
+ fn from(val: ConfigDataType) -> Self {
+ match val {
ConfigDataType::BFloat16 => DataType::BF16,
ConfigDataType::Float16 => DataType::F16,
ConfigDataType::Float32 => DataType::F32,
@@ -64,9 +64,9 @@ impl QuantizationMode {
}
}
-impl Into<DataType> for QuantizationMode {
- fn into(self) -> DataType {
- match self {
+impl From<QuantizationMode> for DataType {
+ fn from(val: QuantizationMode) -> Self {
+ match val {
QuantizationMode::UInt4 => DataType::U4,
QuantizationMode::Int8 => DataType::I8,
QuantizationMode::UInt8 => DataType::U8,
diff --git a/crates/uzu/src/device/data_type.rs b/crates/uzu/src/device/data_type.rs
index 15ef1970f..d0ba5eb78 100644
--- a/crates/uzu/src/device/data_type.rs
+++ b/crates/uzu/src/device/data_type.rs
@@ -1,6 +1,5 @@
use bytemuck::Pod;
use half::{bf16, f16};
-use mpsgraph::DataType as MPSDataType;
use num_traits::NumCast;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Copy, Clone)]
@@ -49,48 +48,6 @@ impl DataType {
}
}
-impl From<MPSDataType> for DataType {
- fn from(dtype: MPSDataType) -> Self {
- match dtype {
- MPSDataType::BFloat16 => DataType::BF16,
- MPSDataType::Float16 => DataType::F16,
- MPSDataType::Float32 => DataType::F32,
- MPSDataType::Int4 => DataType::I4,
- MPSDataType::UInt4 => DataType::U4,
- MPSDataType::Int8 => DataType::I8,
- MPSDataType::UInt8 => DataType::U8,
- MPSDataType::Int16 => DataType::I16,
- MPSDataType::UInt16 => DataType::U16,
- MPSDataType::Int32 => DataType::I32,
- MPSDataType::UInt32 => DataType::U32,
- MPSDataType::Int64 => DataType::I64,
- MPSDataType::UInt64 => DataType::U64,
- _ => panic!("Unsupported MPS data type: {0:?}", dtype),
- }
- }
-}
-
-impl From<DataType> for MPSDataType {
- fn from(dtype: DataType) -> Self {
- match dtype {
- DataType::BF16 => MPSDataType::BFloat16,
- DataType::F16 => MPSDataType::Float16,
- DataType::F32 => MPSDataType::Float32,
- DataType::I4 => MPSDataType::Int4,
- DataType::U4 => MPSDataType::UInt4,
- DataType::I8 => MPSDataType::Int8,
- DataType::U8 => MPSDataType::UInt8,
- DataType::I16 => MPSDataType::Int16,
- DataType::U16 => MPSDataType::UInt16,
- DataType::I32 => MPSDataType::Int32,
- DataType::U32 => MPSDataType::UInt32,
- DataType::I64 => MPSDataType::Int64,
- DataType::U64 => MPSDataType::UInt64,
- _ => panic!("Unsupported MPS data type: {0:?}", dtype),
- }
- }
-}
-
pub trait ArrayElement: NumCast + Pod {
fn data_type() -> DataType;
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 8119aaf15..d58c95617 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -4,7 +4,6 @@ use std::{
use block::ConcreteBlock;
use itertools::{Either, Itertools, izip};
-use mpsgraph::CommandBuffer;
use super::{
LanguageModelGeneratorContext,
@@ -502,8 +501,7 @@ impl LanguageModelGenerator {
}
self.context.reset_command_buffer();
- let root_command_buffer =
- self.context.command_buffer.root_command_buffer();
+ let root_command_buffer = &self.context.command_buffer;
// Wait on previous pass if this is a continuation
if is_continuation {
@@ -611,7 +609,7 @@ impl LanguageModelGenerator {
let block = block.copy();
root_command_buffer.add_completed_handler(&block);
- self.context.command_buffer.commit_and_continue();
+ self.context.command_buffer.commit();
Ok(())
}
@@ -704,8 +702,7 @@ impl LanguageModelGenerator {
);
}
- let root_command_buffer =
- self.context.command_buffer.root_command_buffer().to_owned();
+ let root_command_buffer = self.context.command_buffer.clone();
if !warmup {
if !task.is_prefilling {
@@ -717,9 +714,11 @@ impl LanguageModelGenerator {
}
}
- self.context.command_buffer.commit_and_continue();
+ self.context.command_buffer.commit();
if allow_pre_encode {
+ self.context.reset_command_buffer();
+
let next_task_key: String =
task.encoded_task_key(self.tokens.len() + 1);
@@ -773,11 +772,13 @@ impl LanguageModelGenerator {
suffix_start: Option<usize>,
wait_until_completed: bool,
) {
- let command_buffer = CommandBuffer::from_command_queue(
- &self.context.mtl_context.command_queue,
- );
- let root_command_buffer =
- command_buffer.root_command_buffer().to_owned();
+ let command_buffer = self
+ .context
+ .mtl_context
+ .command_queue
+ .new_command_buffer()
+ .to_owned();
+ let root_command_buffer = command_buffer.to_owned();
{
let mut cache_layers = self.context.cache_layers.borrow_mut();
@@ -792,7 +793,7 @@ impl LanguageModelGenerator {
command_buffer.commit();
if wait_until_completed {
- command_buffer.root_command_buffer().wait_until_completed();
+ command_buffer.wait_until_completed();
}
}
diff --git a/crates/uzu/src/parameters/safetensors_metadata.rs b/crates/uzu/src/parameters/safetensors_metadata.rs
index f7628c223..e5f520b4a 100644
--- a/crates/uzu/src/parameters/safetensors_metadata.rs
+++ b/crates/uzu/src/parameters/safetensors_metadata.rs
@@ -80,9 +80,9 @@ pub enum Dtype {
U64,
}
-impl Into<DataType> for Dtype {
- fn into(self) -> DataType {
- match self {
+impl From<Dtype> for DataType {
+ fn from(val: Dtype) -> Self {
+ match val {
Dtype::F16 => DataType::F16,
Dtype::BF16 => DataType::BF16,
Dtype::F32 => DataType::F32,
@@ -94,7 +94,7 @@ impl Into<DataType> for Dtype {
Dtype::U64 => DataType::U64,
Dtype::BOOL => DataType::I8,
// Add mappings for other types as needed
- _ => panic!("Unsupported dtype: {:?}", self),
+ _ => panic!("Unsupported dtype: {:?}", val),
}
}
}
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index d1c122344..7277373cd 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -1,7 +1,6 @@
#![cfg(any(target_os = "macos", target_os = "ios"))]
-use metal::Device;
-use mpsgraph::CommandBuffer;
+use metal::{CommandBuffer, Device};
use uzu::{
Array, DataType, DeviceContext,
backends::metal::{
@@ -223,10 +222,9 @@ fn run_scenario(
},
};
- let command_buffer =
- CommandBuffer::from_command_queue(&context.command_queue);
+ let command_buffer = context.command_queue.new_command_buffer().to_owned();
- let root_command_buffer = command_buffer.root_command_buffer().to_owned();
+ let root_command_buffer = command_buffer.clone();
layer.update_after_acceptance(
&scenario.accepted_suffix_indices,
scenario.suffix_start,
diff --git a/crates/uzu/tests/split_tensor_datas_test.rs b/crates/uzu/tests/split_tensor_datas_test.rs
deleted file mode 100644
index 2b801da02..000000000
--- a/crates/uzu/tests/split_tensor_datas_test.rs
+++ /dev/null
@@ -1,282 +0,0 @@
-use std::collections::HashMap;
-
-use metal::{Device, MTLResourceOptions};
-use mpsgraph::{
- CommandBuffer, CompilationDescriptor, DataType as MPSDataType,
- Device as MPSDevice, ExecutableExecutionDescriptor, Graph, ShapedType,
- TensorData,
-};
-use ndarray::{Array2, ArrayView2};
-use uzu::{Array, DataType, backends::metal::MetalArray};
-
-fn sequential_f32(len: usize) -> Vec<f32> {
- (0..len).map(|v| v as f32).collect()
-}
-
-fn ndarray_matmul(
- m: usize,
- k: usize,
- n: usize,
- a: &[f32],
- b: &[f32],
-) -> Vec<f32> {
- assert_eq!(a.len(), m * k);
- assert_eq!(b.len(), k * n);
- let a_view = ArrayView2::from_shape((m, k), a).expect("Invalid A shape");
- let b_view = ArrayView2::from_shape((k, n), b).expect("Invalid B shape");
- let c: Array2<f32> = a_view.dot(&b_view);
- let (vec, offset) = c.into_raw_vec_and_offset();
- assert_eq!(offset, Some(0));
- vec
-}
-
-#[test]
-fn test_row_split_no_copy_simple() {
- let device = match Device::system_default() {
- Some(d) => d,
- None => {
- eprintln!("No Metal device; skipping test");
- return;
- },
- };
-
- let shape = [4usize, 4usize];
- let num_elems = shape[0] * shape[1];
- let data = sequential_f32(num_elems);
-
- let buffer_size_bytes = (num_elems * std::mem::size_of::<f32>()) as u64;
- let buffer = device.new_buffer_with_data(
- data.as_ptr() as *const _,
- buffer_size_bytes,
- MTLResourceOptions::StorageModeShared,
- );
-
- let mut array = unsafe { MetalArray::new(buffer, &shape, DataType::F32) };
-
- println!("About to call row_split_no_copy");
- let splits = unsafe { array.row_split_no_copy(&[2, 2]) };
- println!("Got {} splits", splits.len());
- assert_eq!(splits.len(), 2);
-}
-
-#[test]
-fn test_row_split_no_copy_shapes() {
- let device = match Device::system_default() {
- Some(d) => d,
- None => {
- eprintln!("No Metal device; skipping test");
- return;
- },
- };
-
- let shape = [10usize, 6usize];
- let num_elems = shape[0] * shape[1];
- let data = sequential_f32(num_elems);
-
- let buffer_size_bytes = (num_elems * std::mem::size_of::<f32>()) as u64;
- let buffer = device.new_buffer_with_data(
- data.as_ptr() as *const _,
- buffer_size_bytes,
- MTLResourceOptions::StorageModeShared,
- );
-
- let mut array = unsafe { MetalArray::new(buffer, &shape, DataType::F32) };
-
- let splits = unsafe { array.row_split_no_copy(&[2, 5, 3]) };
- assert_eq!(splits.len(), 3);
-
- let expected_shapes =
- vec![[2usize, 6usize], [5usize, 6usize], [3usize, 6usize]];
- for (i, (split_array, expected_shape)) in
- splits.iter().zip(expected_shapes.iter()).enumerate()
- {
- let shape = split_array.shape();
- println!(
- "Split {} has shape {:?}, expected {:?}",
- i, shape, expected_shape
- );
- assert_eq!(shape, *expected_shape, "Shape mismatch for split {}", i);
- }
-
- println!("All row splits have correct shapes!");
-}
-
-#[test]
-fn test_row_split_no_copy_matmul() {
- let device = match Device::system_default() {
- Some(d) => d,
- None => {
- eprintln!("No Metal device; skipping test");
- return;
- },
- };
-
- let a_shape = [5usize, 7usize];
- let b_shape = [7usize, 4usize];
-
- let a_len = a_shape[0] * a_shape[1];
- let b_len = b_shape[0] * b_shape[1];
-
- let a_data = sequential_f32(a_len);
- let b_data: Vec<f32> = (0..b_len).map(|v| v as f32 * 0.5 + 1.0).collect();
-
- let b_buffer_size_bytes = (b_len * std::mem::size_of::<f32>()) as u64;
- let b_buffer = device.new_buffer_with_data(
- b_data.as_ptr() as *const _,
- b_buffer_size_bytes,
- MTLResourceOptions::StorageModeShared,
- );
- let b_td = TensorData::new_with_mtl_buffer(
- &b_buffer,
- &b_shape,
- MPSDataType::Float32,
- None,
- );
-
- let baseline =
- ndarray_matmul(a_shape[0], a_shape[1], b_shape[1], &a_data, &b_data);
-
- let a_buffer_size_bytes = (a_len * std::mem::size_of::<f32>()) as u64;
- let a_buffer = device.new_buffer_with_data(
- a_data.as_ptr() as *const _,
- a_buffer_size_bytes,
- MTLResourceOptions::StorageModeShared,
- );
- let mut a_array =
- unsafe { MetalArray::new(a_buffer, &a_shape, DataType::F32) };
-
- let split_lengths = [2usize, 3usize];
- let mut splits = unsafe { a_array.row_split_no_copy(&split_lengths) };
- assert_eq!(splits.len(), split_lengths.len());
-
- let mut combined_c = Vec::with_capacity(baseline.len());
- let mut offset = 0usize;
- for (split_array, &rows) in splits.iter_mut().zip(split_lengths.iter()) {
- let slice_start = offset * a_shape[1];
- let slice_end = slice_start + rows * a_shape[1];
- let slice_cpu = &a_data[slice_start..slice_end];
- let expected =
- ndarray_matmul(rows, a_shape[1], b_shape[1], slice_cpu, &b_data);
-
- let graph = Graph::new();
-
- let shape_a_isize = [rows as isize, a_shape[1] as isize];
- let a_ph = graph.placeholder(
- Some(&shape_a_isize),
- MPSDataType::Float32,
- Some("A"),
- );
-
- let shape_b_isize = [a_shape[1] as isize, b_shape[1] as isize];
- let b_ph = graph.placeholder(
- Some(&shape_b_isize),
- MPSDataType::Float32,
- Some("B"),
- );
-
- let c_tensor =
- graph.matrix_multiplication(&a_ph, &b_ph, Some("MatMul"));
-
- let mps_device = MPSDevice::with_device(&device);
- let compilation_descriptor = CompilationDescriptor::new();
-
- let a_shaped_type = ShapedType::new_with_shape_data_type(
- Some(&shape_a_isize),
- MPSDataType::Float32,
- );
- let b_shaped_type = ShapedType::new_with_shape_data_type(
- Some(&shape_b_isize),
- MPSDataType::Float32,
- );
-
- let mut compile_feeds = HashMap::new();
- compile_feeds.insert(a_ph.as_ref(), a_shaped_type.as_ref());
- compile_feeds.insert(b_ph.as_ref(), b_shaped_type.as_ref());
-
- let targets = [c_tensor.as_ref()];
- let executable = graph.compile(
- &mps_device,
- &compile_feeds,
- &targets,
- None,
- Some(&compilation_descriptor),
- );
-
- let result_size = rows * b_shape[1];
- let result_buffer_size =
- (result_size * std::mem::size_of::<f32>()) as u64;
- let result_buffer = device.new_buffer(
- result_buffer_size,
- MTLResourceOptions::StorageModeShared,
- );
- let result_shape_usize = [rows, b_shape[1]];
- let result_td = TensorData::new_with_mtl_buffer(
- &result_buffer,
- &result_shape_usize,
- MPSDataType::Float32,
- None,
- );
-
- let split_td = unsafe { split_array.to_mps_tensor_data() };
-
- let inputs = [split_td.as_ref(), b_td.as_ref()];
- let outputs = [result_td.as_ref()];
-
- let command_queue = device.new_command_queue();
- let command_buffer = CommandBuffer::from_command_buffer(
- &command_queue.new_command_buffer().to_owned(),
- );
-
- let exec_desc = ExecutableExecutionDescriptor::new();
- exec_desc.set_wait_until_completed(true);
-
- let _ = executable.encode_to_command_buffer(
- &command_buffer,
- &inputs,
- Some(&outputs),
- Some(&exec_desc),
- );
-
- command_buffer.commit();
- command_buffer.command_buffer().wait_until_completed();
-
- let gpu_result = unsafe {
- let ptr = result_buffer.contents() as *const f32;
- std::slice::from_raw_parts(ptr, result_size).to_vec()
- };
-
- for (i, (&exp, &got)) in
- expected.iter().zip(gpu_result.iter()).enumerate()
- {
- assert!(
- (exp - got).abs() < 1e-4,
- "Mismatch rows slice {} idx {}: expected {}, got {}",
- rows,
- i,
- exp,
- got
- );
- }
-
- combined_c.extend_from_slice(&expected);
-
- offset += rows;
- }
-
- assert_eq!(combined_c.len(), baseline.len());
- for (i, (expected, got)) in
- baseline.iter().zip(combined_c.iter()).enumerate()
- {
- assert!(
- (expected - got).abs() < 1e-6,
- "Mismatch at index {}: expected {}, got {}",
- i,
- expected,
- got
- );
- }
-
- println!(
- "Matrix multiplication results match for split and unsplit tensors!"
- );
-}