A high-performance inference engine for AI models
decouple linear into a mod
eugened13297a9ccbe
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsnew file mode 100644index 000000000..155324388--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs@@ -0,0 +1,231 @@+use std::{cell::RefCell, rc::Rc};++use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use crate::{+ DataType,+ backends::metal::{+ MTLContext, MTLError,+ encodable_block::{EncodableBlock, EncodingParameters},+ forward_pass::{ArrayId, ForwardPassState},+ kernel::{+ KernelDataType, TensorAddBias,+ matmul::{MatmulArguments, MatmulKernel},+ },+ },+ device::array::Array,+ parameters::ParameterTree,+};++pub struct FullPrecisionLinear {+ kernel: RefCell<MatmulKernel>,+ bias_add_kernel: Option<TensorAddBias>,+ biases_buffer: Option<MTLBuffer>,+ weights_buffer: MTLBuffer,+ input_dim: usize,+ output_dim: usize,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+}++impl FullPrecisionLinear {+ pub fn new(+ mtl_context: &MTLContext,+ precision: DataType,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, MTLError> {+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for full precision linear kernel: {:?}",+ precision+ )));+ }++ let mut weights = parameter_tree.leaf("weights").map_err(|e| {+ MTLError::Generic(format!("Failed to load weights: {:?}", e))+ })?;++ let w_shape = weights.shape();+ if w_shape != [output_dim, input_dim] {+ return Err(MTLError::Generic(format!(+ "Unexpected weights shape: got {:?}, expected [{}, {}]",+ w_shape, output_dim, input_dim+ )));+ }++ if weights.data_type() != precision {+ return Err(MTLError::Generic(format!(+ "Weights dtype mismatch: got {:?}, expected {:?}",+ weights.data_type(),+ precision+ )));+ }++ let weights_buffer: MTLBuffer =+ unsafe { weights.mtl_buffer() }.to_owned();++ let (bias_add_kernel, biases_buffer) =+ match parameter_tree.leaf("biases") {+ Ok(mut biases) => {+ if biases.shape() != [output_dim] {+ return Err(MTLError::Generic(format!(+ "Bias shape mismatch: got {:?}, expected [{:?}]",+ biases.shape(),+ output_dim+ )));+ }+ if biases.data_type() != precision {+ return Err(MTLError::Generic(format!(+ "Bias dtype mismatch: got {:?}, expected {:?}",+ biases.data_type(),+ precision+ )));+ }+ let bias_add_kernel = Some(TensorAddBias::new(+ mtl_context,+ KernelDataType::from(precision),+ )?);+ let biases_buffer: MTLBuffer =+ unsafe { biases.mtl_buffer() }.to_owned();+ (bias_add_kernel, Some(biases_buffer))+ },+ Err(_) => (None, None),+ };++ let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;++ Ok(Self {+ kernel: RefCell::new(kernel),+ bias_add_kernel,+ biases_buffer,+ weights_buffer,+ input_dim,+ output_dim,+ input_array_id,+ output_array_id,+ })+ }+}++impl EncodableBlock for FullPrecisionLinear {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let batch_size = state.active_suffix_length();++ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = MatmulArguments {+ a: input_buffer,+ b: &self.weights_buffer,+ d: output_buffer,+ batch: batch_size as i32,+ input_dim: self.input_dim as i32,+ output_dim: self.output_dim as i32,+ lda: self.input_dim as i32,+ ldb: self.input_dim as i32,+ ldd: self.output_dim as i32,+ batch_count: 1,+ };++ self.kernel+ .borrow_mut()+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");++ encoder.end_encoding();++ if let (Some(bias_add), Some(bias_buf)) =+ (&self.bias_add_kernel, &self.biases_buffer)+ {+ let total_len = batch_size * self.output_dim;+ let retained_cb = root_command_buffer.to_owned();+ bias_add.encode_into_command_buffer(+ &output_buffer,+ bias_buf,+ &output_buffer,+ self.output_dim,+ total_len,+ &retained_cb,+ parameters.predicate,+ );+ }++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let batch_size = state.active_suffix_length();++ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let args = MatmulArguments {+ a: input_buffer,+ b: &self.weights_buffer,+ d: output_buffer,+ batch: batch_size as i32,+ input_dim: self.input_dim as i32,+ output_dim: self.output_dim as i32,+ lda: self.input_dim as i32,+ ldb: self.input_dim as i32,+ ldd: self.output_dim as i32,+ batch_count: 1,+ };++ self.kernel+ .borrow_mut()+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");++ if let (Some(bias_add), Some(bias_buf)) =+ (&self.bias_add_kernel, &self.biases_buffer)+ {+ let total_len = batch_size * self.output_dim;+ bias_add.encode_with_encoder(+ &output_buffer,+ bias_buf,+ &output_buffer,+ self.output_dim,+ total_len,+ encoder,+ parameters.predicate,+ );+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rsnew file mode 100644index 000000000..139ecef36--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs@@ -0,0 +1,5 @@+mod full_precision;+mod quantized;++pub use full_precision::FullPrecisionLinear;+pub use quantized::QuantizedLinear;diff --git a/crates/uzu/src/backends/metal/encodable_block/linear.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rssimilarity index 62%rename from crates/uzu/src/backends/metal/encodable_block/linear.rsrename to crates/uzu/src/backends/metal/encodable_block/linear/quantized.rsindex 17293ca29..9fab0314e 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs@@ -1,19 +1,16 @@-//! Linear encodables implemented with Metal kernels.--use std::{cell::RefCell, rc::Rc};+use std::rc::Rc;use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::{EncodableBlock, EncodingParameters};use crate::{- Array, DataType,+ DataType,backends::metal::{MTLContext, MTLError,+ encodable_block::{EncodableBlock, EncodingParameters},forward_pass::{ArrayId, ForwardPassState},kernel::{KernelDataType, TensorAddBias,- matmul::{MatmulArguments, MatmulKernel},quant_matmul::{QuantizationType, QuantizedMatmulArguments,QuantizedMatmulKernel,@@ -21,221 +18,10 @@ use crate::{},},config::QuantizationConfig,+ device::array::Array,parameters::ParameterTree,};-pub struct FullPrecisionLinear {- kernel: RefCell<MatmulKernel>,- bias_add_kernel: Option<TensorAddBias>,- biases_buffer: Option<MTLBuffer>,- weights_buffer: MTLBuffer,- input_dim: usize,- output_dim: usize,- input_array_id: ArrayId,- output_array_id: ArrayId,-}--impl FullPrecisionLinear {- pub fn new(- mtl_context: &MTLContext,- precision: DataType,- input_dim: usize,- output_dim: usize,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- input_array_id: ArrayId,- output_array_id: ArrayId,- ) -> Result<Self, MTLError> {- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported data type for full precision linear kernel: {:?}",- precision- )));- }-- let mut weights = parameter_tree.leaf("weights").map_err(|e| {- MTLError::Generic(format!("Failed to load weights: {:?}", e))- })?;-- let w_shape = weights.shape();- if w_shape != [output_dim, input_dim] {- return Err(MTLError::Generic(format!(- "Unexpected weights shape: got {:?}, expected [{}, {}]",- w_shape, output_dim, input_dim- )));- }-- if weights.data_type() != precision {- return Err(MTLError::Generic(format!(- "Weights dtype mismatch: got {:?}, expected {:?}",- weights.data_type(),- precision- )));- }-- let weights_buffer: MTLBuffer =- unsafe { weights.mtl_buffer() }.to_owned();-- let (bias_add_kernel, biases_buffer) =- match parameter_tree.leaf("biases") {- Ok(mut biases) => {- if biases.shape() != [output_dim] {- return Err(MTLError::Generic(format!(- "Bias shape mismatch: got {:?}, expected [{:?}]",- biases.shape(),- output_dim- )));- }- if biases.data_type() != precision {- return Err(MTLError::Generic(format!(- "Bias dtype mismatch: got {:?}, expected {:?}",- biases.data_type(),- precision- )));- }- let bias_add_kernel = Some(TensorAddBias::new(- mtl_context,- KernelDataType::from(precision),- )?);- let biases_buffer: MTLBuffer =- unsafe { biases.mtl_buffer() }.to_owned();- (bias_add_kernel, Some(biases_buffer))- },- Err(_) => (None, None),- };-- let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;-- Ok(Self {- kernel: RefCell::new(kernel),- bias_add_kernel,- biases_buffer,- weights_buffer,- input_dim,- output_dim,- input_array_id,- output_array_id,- })- }-}--impl EncodableBlock for FullPrecisionLinear {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &MPSCommandBuffer,- parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);- let batch_size = state.active_suffix_length();-- let mut input_array_mut = arrays[0].borrow_mut();- let mut output_array_mut = arrays[1].borrow_mut();-- let input_buffer = unsafe { input_array_mut.mtl_buffer() };- let output_buffer = unsafe { output_array_mut.mtl_buffer() };-- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();-- let args = MatmulArguments {- a: input_buffer,- b: &self.weights_buffer,- d: output_buffer,- batch: batch_size as i32,- input_dim: self.input_dim as i32,- output_dim: self.output_dim as i32,- lda: self.input_dim as i32,- ldb: self.input_dim as i32,- ldd: self.output_dim as i32,- batch_count: 1,- };-- self.kernel- .borrow_mut()- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode matmul kernel");-- encoder.end_encoding();-- if let (Some(bias_add), Some(bias_buf)) =- (&self.bias_add_kernel, &self.biases_buffer)- {- let total_len = batch_size * self.output_dim;- let retained_cb = root_command_buffer.to_owned();- bias_add.encode_into_command_buffer(- &output_buffer,- bias_buf,- &output_buffer,- self.output_dim,- total_len,- &retained_cb,- parameters.predicate,- );- }-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState,- encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);- let batch_size = state.active_suffix_length();-- let mut input_array_mut = arrays[0].borrow_mut();- let mut output_array_mut = arrays[1].borrow_mut();-- let input_buffer = unsafe { input_array_mut.mtl_buffer() };- let output_buffer = unsafe { output_array_mut.mtl_buffer() };-- let args = MatmulArguments {- a: input_buffer,- b: &self.weights_buffer,- d: output_buffer,- batch: batch_size as i32,- input_dim: self.input_dim as i32,- output_dim: self.output_dim as i32,- lda: self.input_dim as i32,- ldb: self.input_dim as i32,- ldd: self.output_dim as i32,- batch_count: 1,- };-- self.kernel- .borrow_mut()- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode matmul kernel");-- if let (Some(bias_add), Some(bias_buf)) =- (&self.bias_add_kernel, &self.biases_buffer)- {- let total_len = batch_size * self.output_dim;- bias_add.encode_with_encoder(- &output_buffer,- bias_buf,- &output_buffer,- self.output_dim,- total_len,- encoder,- parameters.predicate,- );- }- }-}-pub struct QuantizedLinear {kernel: QuantizedMatmulKernel,bias_add_kernel: Option<TensorAddBias>,diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 2acd7b814..43ff39948 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -35,8 +35,8 @@ pub use attention::Attention;pub use classifier_layer::ClassifierLayer;pub use decoder::Decoder;pub use embedding::{- FullPrecisionEmbeddingReadout, QuantizedEmbeddingError,- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,+ FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout,+ QuantizedEmbeddingError, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,};pub use encoding_parameters::EncodingParameters;pub use layer::LayerExecutables;diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 53ed69431..9ef459df6 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -7,9 +7,9 @@ use mpsgraph::{use objc2::rc::{Retained, autoreleasepool};use super::{- EncodableBlock, FullPrecisionEmbeddingReadout, FullPrecisionLinear,- MlpBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,- QuantizedLinear,+ EncodableBlock, FullPrecisionEmbeddingLookup,+ FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock, MoeBlock,+ QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,@@ -23,10 +23,7 @@ use crate::{},kernel::mlp::MlpGateActMulEncodable,},- config::{- DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig,- QuantizationConfig,- },+ config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},parameters::ParameterTree,};@@ -51,16 +48,17 @@ pub fn linear_block<const N: usize>(match config {LinearConfig::Quantized(quant_config)| LinearConfig::MLXQuantized(quant_config) => {- return quantized_linear_block_custom(+ let block = QuantizedLinear::new(+ context,quant_config,input_dim,out_sum,- context,parameter_tree,input_array_id,output_array_id,)- .unwrap();+ .expect("Failed to create quantized linear kernel block");+ return Box::new(block);},LinearConfig::FullPrecision {precision,@@ -126,27 +124,6 @@ pub fn linear_block<const N: usize>(})}-pub fn quantized_linear_block_custom(- config: &QuantizationConfig,- input_dim: usize,- output_dim: usize,- context: &MTLContext,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- input_array_id: ArrayId,- output_array_id: ArrayId,-) -> Result<Box<dyn EncodableBlock>, crate::backends::metal::MTLError> {- let block = QuantizedLinear::new(- context,- config,- input_dim,- output_dim,- parameter_tree,- input_array_id,- output_array_id,- )?;- Ok(Box::new(block))-}-pub fn mlp_block(config: &MLPConfig,model_dim: usize,@@ -266,146 +243,65 @@ pub fn embed_block(compilation_descriptor: &CompilationDescriptor,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Box<dyn EncodableBlock> {- let graph = Graph::new();-- let input_shape = [-1 as isize];- let input_data_type = DataType::U64;- let input_placeholder = placeholder(&graph, &input_shape, input_data_type);- let input_shaped_type = shaped_type(&input_shape, input_data_type);-match &config.embedding_config {EmbeddingConfig::Tied {+ common,precision,- ..} => {- let weights_shape =- [config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = (*precision).into();- let weights_shaped_type =- shaped_type(&weights_shape, weights_data_type);- let weights_placeholder =- placeholder(&graph, &weights_shape, weights_data_type);+ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");- let output = embeddings_embed_subgraph(- &graph,- &config.embedding_config,- &input_placeholder,- &weights_placeholder,+ let block = FullPrecisionEmbeddingLookup::new(+ context,+ (*precision).into(),+ config.vocab_size,+ config.model_dim,+ common.input_scale,+ &embeddings_tree,)- .unwrap();-- let feeds = HashMap::from([- (&*input_placeholder, &*input_shaped_type),- (&*weights_placeholder, &*weights_shaped_type),- ]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(&compilation_descriptor),- );-- let block = MPSGraphBlock::new(- executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]- .into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- ),- );+ .expect("Failed to create full precision embedding lookup");Box::new(block)},EmbeddingConfig::Untied {+ common,precision,- ..} => {- let weights_shape =- [config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = (*precision).into();- let weights_shaped_type =- shaped_type(&weights_shape, weights_data_type);- let weights_placeholder =- placeholder(&graph, &weights_shape, weights_data_type);+ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");- let output = embeddings_embed_subgraph(- &graph,- &config.embedding_config,- &input_placeholder,- &weights_placeholder,+ let block = FullPrecisionEmbeddingLookup::new(+ context,+ (*precision).into(),+ config.vocab_size,+ config.model_dim,+ common.input_scale,+ &embeddings_tree,)- .unwrap();-- let feeds = HashMap::from([- (&*input_placeholder, &*input_shaped_type),- (&*weights_placeholder, &*weights_shaped_type),- ]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(&compilation_descriptor),- );-- let block = MPSGraphBlock::new(- executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]- .into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- ),- );+ .expect("Failed to create full precision embedding lookup");Box::new(block)},EmbeddingConfig::MLXSemiQuantizedUntied {+ common,activation_precision,..} => {- let weights_shape =- [config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = (*activation_precision).into();- let weights_shaped_type =- shaped_type(&weights_shape, weights_data_type);- let weights_placeholder =- placeholder(&graph, &weights_shape, weights_data_type);+ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");- let output = embeddings_embed_subgraph(- &graph,- &config.embedding_config,- &input_placeholder,- &weights_placeholder,+ let block = FullPrecisionEmbeddingLookup::new(+ context,+ (*activation_precision).into(),+ config.vocab_size,+ config.model_dim,+ common.input_scale,+ &embeddings_tree,)- .unwrap();-- let feeds = HashMap::from([- (&*input_placeholder, &*input_shaped_type),- (&*weights_placeholder, &*weights_shaped_type),- ]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(&compilation_descriptor),- );-- let block = MPSGraphBlock::new(- executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]- .into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- ),- );+ .expect("Failed to create full precision embedding lookup");Box::new(block)},@@ -413,6 +309,13 @@ pub fn embed_block(embedding_quantization_mode: _,..} => {+ let graph = Graph::new();++ let input_shape = [-1 as isize];+ let input_data_type = DataType::U64;+ let input_placeholder =+ placeholder(&graph, &input_shape, input_data_type);+ let input_shaped_type = shaped_type(&input_shape, input_data_type);let weights_shape =[config.vocab_size as isize, (config.model_dim / 2) as isize];let weights_data_type = DataType::U8;