A high-performance inference engine for AI models
wip
eugenefef0ad5ad589
diff --git a/.gitignore b/.gitignoreindex afdab63c4..b79fa8afe 100644--- a/.gitignore+++ b/.gitignore@@ -28,8 +28,9 @@ target/models/external/-# Local-only dev example-crates/uzu/examples/quick_chat.rs+# GH+.github/*.md-# Local-only dev example: structured chat output validator+# Local-only dev examples+crates/uzu/examples/quick_chat.rscrates/uzu/examples/chat_output_validator.rsdiff --git a/crates/uzu/src/backends/metal/encodable_block/linear.rs b/crates/uzu/src/backends/metal/encodable_block/linear.rsindex 4bb8dc2b0..17293ca29 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear.rs@@ -1,6 +1,6 @@-//! Quantized linear (matmul) encodable.+//! Linear encodables implemented with Metal kernels.-use std::rc::Rc;+use std::{cell::RefCell, rc::Rc};use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};use mpsgraph::CommandBuffer as MPSCommandBuffer;@@ -13,6 +13,7 @@ use crate::{forward_pass::{ArrayId, ForwardPassState},kernel::{KernelDataType, TensorAddBias,+ matmul::{MatmulArguments, MatmulKernel},quant_matmul::{QuantizationType, QuantizedMatmulArguments,QuantizedMatmulKernel,@@ -23,6 +24,218 @@ use crate::{parameters::ParameterTree,};+pub struct FullPrecisionLinear {+ kernel: RefCell<MatmulKernel>,+ bias_add_kernel: Option<TensorAddBias>,+ biases_buffer: Option<MTLBuffer>,+ weights_buffer: MTLBuffer,+ input_dim: usize,+ output_dim: usize,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+}++impl FullPrecisionLinear {+ pub fn new(+ mtl_context: &MTLContext,+ precision: DataType,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, MTLError> {+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for full precision linear kernel: {:?}",+ precision+ )));+ }++ let mut weights = parameter_tree.leaf("weights").map_err(|e| {+ MTLError::Generic(format!("Failed to load weights: {:?}", e))+ })?;++ let w_shape = weights.shape();+ if w_shape != [output_dim, input_dim] {+ return Err(MTLError::Generic(format!(+ "Unexpected weights shape: got {:?}, expected [{}, {}]",+ w_shape, output_dim, input_dim+ )));+ }++ if weights.data_type() != precision {+ return Err(MTLError::Generic(format!(+ "Weights dtype mismatch: got {:?}, expected {:?}",+ weights.data_type(),+ precision+ )));+ }++ let weights_buffer: MTLBuffer =+ unsafe { weights.mtl_buffer() }.to_owned();++ let (bias_add_kernel, biases_buffer) =+ match parameter_tree.leaf("biases") {+ Ok(mut biases) => {+ if biases.shape() != [output_dim] {+ return Err(MTLError::Generic(format!(+ "Bias shape mismatch: got {:?}, expected [{:?}]",+ biases.shape(),+ output_dim+ )));+ }+ if biases.data_type() != precision {+ return Err(MTLError::Generic(format!(+ "Bias dtype mismatch: got {:?}, expected {:?}",+ biases.data_type(),+ precision+ )));+ }+ let bias_add_kernel = Some(TensorAddBias::new(+ mtl_context,+ KernelDataType::from(precision),+ )?);+ let biases_buffer: MTLBuffer =+ unsafe { biases.mtl_buffer() }.to_owned();+ (bias_add_kernel, Some(biases_buffer))+ },+ Err(_) => (None, None),+ };++ let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;++ Ok(Self {+ kernel: RefCell::new(kernel),+ bias_add_kernel,+ biases_buffer,+ weights_buffer,+ input_dim,+ output_dim,+ input_array_id,+ output_array_id,+ })+ }+}++impl EncodableBlock for FullPrecisionLinear {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let batch_size = state.active_suffix_length();++ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = MatmulArguments {+ a: input_buffer,+ b: &self.weights_buffer,+ d: output_buffer,+ batch: batch_size as i32,+ input_dim: self.input_dim as i32,+ output_dim: self.output_dim as i32,+ lda: self.input_dim as i32,+ ldb: self.input_dim as i32,+ ldd: self.output_dim as i32,+ batch_count: 1,+ };++ self.kernel+ .borrow_mut()+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");++ encoder.end_encoding();++ if let (Some(bias_add), Some(bias_buf)) =+ (&self.bias_add_kernel, &self.biases_buffer)+ {+ let total_len = batch_size * self.output_dim;+ let retained_cb = root_command_buffer.to_owned();+ bias_add.encode_into_command_buffer(+ &output_buffer,+ bias_buf,+ &output_buffer,+ self.output_dim,+ total_len,+ &retained_cb,+ parameters.predicate,+ );+ }++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let batch_size = state.active_suffix_length();++ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let args = MatmulArguments {+ a: input_buffer,+ b: &self.weights_buffer,+ d: output_buffer,+ batch: batch_size as i32,+ input_dim: self.input_dim as i32,+ output_dim: self.output_dim as i32,+ lda: self.input_dim as i32,+ ldb: self.input_dim as i32,+ ldd: self.output_dim as i32,+ batch_count: 1,+ };++ self.kernel+ .borrow_mut()+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");++ if let (Some(bias_add), Some(bias_buf)) =+ (&self.bias_add_kernel, &self.biases_buffer)+ {+ let total_len = batch_size * self.output_dim;+ bias_add.encode_with_encoder(+ &output_buffer,+ bias_buf,+ &output_buffer,+ self.output_dim,+ total_len,+ encoder,+ parameters.predicate,+ );+ }+ }+}+pub struct QuantizedLinear {kernel: QuantizedMatmulKernel,bias_add_kernel: Option<TensorAddBias>,diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 7497d8f95..5577febb6 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -40,7 +40,7 @@ pub use embedding::{};pub use encoding_parameters::EncodingParameters;pub use layer::LayerExecutables;-pub use linear::QuantizedLinear;+pub use linear::{FullPrecisionLinear, QuantizedLinear};pub(crate) use mamba_mixer::MambaMixer;pub use mlp::MlpBlock;pub use moe_block::{MoeBlock, SharedMoeWeights};diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 12f439f33..013620942 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -7,8 +7,8 @@ use mpsgraph::{use objc2::rc::{Retained, autoreleasepool};use super::{- EncodableBlock, MlpBlock, MoeBlock, QuantizedEmbeddingLookup,- QuantizedEmbeddingReadout, QuantizedLinear,+ EncodableBlock, FullPrecisionLinear, MlpBlock, MoeBlock,+ QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,@@ -37,7 +37,7 @@ fn make_execution_descriptor() -> Retained<ExecutableExecutionDescriptor> {pub fn linear_block<const N: usize>(config: &LinearConfig,- has_biases: bool,+ _has_biases: bool,input_dim: usize,output_dims: [usize; N],context: &MTLContext,@@ -46,31 +46,39 @@ pub fn linear_block<const N: usize>(output_array_id: ArrayId,compilation_descriptor: &CompilationDescriptor,) -> Box<dyn EncodableBlock> {- if let LinearConfig::Quantized(quant_config) = config {- let out_sum: usize = output_dims.iter().sum();- return quantized_linear_block_custom(- quant_config,- input_dim,- out_sum,- context,- parameter_tree,- input_array_id,- output_array_id,- )- .unwrap();- }- if let LinearConfig::MLXQuantized(quant_config) = config {- let out_sum: usize = output_dims.iter().sum();- return quantized_linear_block_custom(- quant_config,- input_dim,- out_sum,- context,- parameter_tree,- input_array_id,- output_array_id,- )- .unwrap();+ let out_sum: usize = output_dims.iter().sum();+ match config {+ LinearConfig::Quantized(quant_config)+ | LinearConfig::MLXQuantized(quant_config) => {+ return quantized_linear_block_custom(+ quant_config,+ input_dim,+ out_sum,+ context,+ parameter_tree,+ input_array_id,+ output_array_id,+ )+ .unwrap();+ },+ LinearConfig::FullPrecision {+ precision,+ } => {+ let block = FullPrecisionLinear::new(+ context,+ (*precision).into(),+ input_dim,+ out_sum,+ parameter_tree,+ input_array_id,+ output_array_id,+ )+ .expect("Failed to create full precision linear kernel block");+ return Box::new(block);+ },+ LinearConfig::QLoRA {+ ..+ } => {},}autoreleasepool(|_| {@@ -86,7 +94,7 @@ pub fn linear_block<const N: usize>(config,input_dim,output_dims,- has_biases,+ true,&input_placeholder,parameter_tree,)@@ -543,6 +551,10 @@ pub fn readout_block(precision,..} => {+ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");+let weights_shape =[config.vocab_size as isize, config.model_dim as isize];let weights_data_type: DataType = precision.into();@@ -553,6 +565,13 @@ pub fn readout_block(let weights_transposed =graph.transpose(&weights_placeholder, &[1, 0], None);+ let input_shape = [-1 as isize, config.model_dim as isize];+ let input_data_type: DataType =+ config.output_norm_config.scale_precision.into();+ let input_placeholder =+ placeholder(&graph, &input_shape, input_data_type);+ let input_shaped_type = shaped_type(&input_shape, input_data_type);+let output = embeddings_readout_subgraph(&graph,&input_placeholder,@@ -570,7 +589,7 @@ pub fn readout_block(&feeds,&[&output],None,- Some(&compilation_descriptor),+ Some(compilation_descriptor),);let block = MPSGraphBlock::new(@@ -589,6 +608,10 @@ pub fn readout_block(precision,..} => {+ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");+ // Use MPSGraph readout pathlet weights_shape =[config.vocab_size as isize, config.model_dim as isize];let weights_data_type: DataType = precision.into();@@ -599,6 +622,13 @@ pub fn readout_block(let weights_transposed =graph.transpose(&weights_placeholder, &[1, 0], None);+ let input_shape = [-1 as isize, config.model_dim as isize];+ let input_data_type: DataType =+ config.output_norm_config.scale_precision.into();+ let input_placeholder =+ placeholder(&graph, &input_shape, input_data_type);+ let input_shaped_type = shaped_type(&input_shape, input_data_type);+let output = embeddings_readout_subgraph(&graph,&input_placeholder,@@ -616,7 +646,7 @@ pub fn readout_block(&feeds,&[&output],None,- Some(&compilation_descriptor),+ Some(compilation_descriptor),);let block = MPSGraphBlock::new(@@ -659,7 +689,15 @@ pub fn readout_block(embedding_quantization_mode: _,..} => {- // Packed U8 weights with shape [vocab_size, model_dim / 2]+ let graph = Graph::new();++ let input_shape = [-1 as isize, config.model_dim as isize];+ let input_data_type: DataType =+ config.output_norm_config.scale_precision.into();+ let input_placeholder =+ placeholder(&graph, &input_shape, input_data_type);+ let input_shaped_type = shaped_type(&input_shape, input_data_type);+let weights_shape =[config.vocab_size as isize, (config.model_dim / 2) as isize];let weights_data_type = DataType::U8;@@ -701,7 +739,7 @@ pub fn readout_block(&feeds,&[&output],None,- Some(&compilation_descriptor),+ Some(compilation_descriptor),);let block = MPSGraphBlock::new(diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rsindex 4adbc459a..f5513f2af 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs@@ -1,14 +1,20 @@-use std::mem::size_of;+use std::{mem::size_of, rc::Rc};use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef,ComputePipelineState as MTLComputePipelineState, MTLSize,};-use super::super::MTLContext;+use super::super::{MTLContext, kernel::matmul::MatmulArguments};use crate::{- DataType, backends::metal::encodable_block::QuantizedEmbeddingError,+ DataType,+ backends::metal::{+ ForwardPassState, MTLError, encodable_block::QuantizedEmbeddingError,+ forward_pass::ArrayId,+ },config::QuantizationMode,+ device::array::Array,+ parameters::ParameterTree,};pub struct QuantizedEmbeddingLookupKernel {@@ -119,3 +125,102 @@ impl QuantizedEmbeddingLookupKernel {Ok(())}}++pub struct FullPrecisionReadoutKernelBlock {+ kernel: std::cell::RefCell<super::super::kernel::matmul::MatmulKernel>,+ weights_buffer: MTLBuffer,+ vocab_size: usize,+ model_dim: usize,+}++impl FullPrecisionReadoutKernelBlock {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for full precision readout: {:?}",+ data_type+ )));+ }++ let mut weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {+ MTLError::Generic(format!("Failed to load weights: {:?}", e))+ })?,+ };++ if weights.shape() != [vocab_size, model_dim] {+ return Err(MTLError::Generic(format!(+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim+ )));+ }++ if weights.data_type() != data_type {+ return Err(MTLError::Generic(format!(+ "Weights dtype mismatch: got {:?}, expected {:?}",+ weights.data_type(),+ data_type+ )));+ }++ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };++ let kernel = super::super::kernel::matmul::MatmulKernel::new(+ mtl_context,+ data_type,+ false,+ true,+ )?;++ Ok(Self {+ kernel: std::cell::RefCell::new(kernel),+ weights_buffer,+ vocab_size,+ model_dim,+ })+ }+}++impl FullPrecisionReadoutKernelBlock {+ fn encode_impl(+ &self,+ state: &mut ForwardPassState,+ encoder: &metal::ComputeCommandEncoderRef,+ ) {+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);+ let batch_size = state.active_suffix_length();+ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let args = MatmulArguments {+ a: input_buffer,+ b: &self.weights_buffer,+ d: output_buffer,+ batch: batch_size as i32,+ input_dim: self.model_dim as i32,+ output_dim: self.vocab_size as i32,+ lda: self.model_dim as i32,+ ldb: self.model_dim as i32, // B is [vocab_size, model_dim], stride is model_dim+ ldd: self.vocab_size as i32,+ batch_count: 1,+ };++ self.kernel+ .borrow_mut()+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode full precision readout kernel");+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex bcf40230c..0528c0967 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -3,6 +3,7 @@ mod data_type;pub mod embedding;pub mod kv_cache_update;pub mod layer_norm;+pub mod matmul;pub mod mask_update;pub mod media_kernels;pub mod mlp;@@ -28,6 +29,7 @@ pub use attention::{pub use data_type::KernelDataType;pub use kv_cache_update::KVCacheUpdate;pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};+pub use matmul::{MatmulArguments, MatmulKernel};pub use mask_update::MaskUpdateKernel;pub use moe::{MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,