Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

wip

eugenefef0ad5ad589
diff --git a/.gitignore b/.gitignore
index afdab63c4..b79fa8afe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,8 +28,9 @@ target/
models/
external/
-# Local-only dev example
-crates/uzu/examples/quick_chat.rs
+# GH
+.github/*.md
-# Local-only dev example: structured chat output validator
+# Local-only dev examples
+crates/uzu/examples/quick_chat.rs
crates/uzu/examples/chat_output_validator.rs
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear.rs b/crates/uzu/src/backends/metal/encodable_block/linear.rs
index 4bb8dc2b0..17293ca29 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear.rs
@@ -1,6 +1,6 @@
-//! Quantized linear (matmul) encodable.
+//! Linear encodables implemented with Metal kernels.
-use std::rc::Rc;
+use std::{cell::RefCell, rc::Rc};
use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
@@ -13,6 +13,7 @@ use crate::{
forward_pass::{ArrayId, ForwardPassState},
kernel::{
KernelDataType, TensorAddBias,
+ matmul::{MatmulArguments, MatmulKernel},
quant_matmul::{
QuantizationType, QuantizedMatmulArguments,
QuantizedMatmulKernel,
@@ -23,6 +24,218 @@ use crate::{
parameters::ParameterTree,
};
+pub struct FullPrecisionLinear {
+ kernel: RefCell<MatmulKernel>,
+ bias_add_kernel: Option<TensorAddBias>,
+ biases_buffer: Option<MTLBuffer>,
+ weights_buffer: MTLBuffer,
+ input_dim: usize,
+ output_dim: usize,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+}
+
+impl FullPrecisionLinear {
+ pub fn new(
+ mtl_context: &MTLContext,
+ precision: DataType,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, MTLError> {
+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for full precision linear kernel: {:?}",
+ precision
+ )));
+ }
+
+ let mut weights = parameter_tree.leaf("weights").map_err(|e| {
+ MTLError::Generic(format!("Failed to load weights: {:?}", e))
+ })?;
+
+ let w_shape = weights.shape();
+ if w_shape != [output_dim, input_dim] {
+ return Err(MTLError::Generic(format!(
+ "Unexpected weights shape: got {:?}, expected [{}, {}]",
+ w_shape, output_dim, input_dim
+ )));
+ }
+
+ if weights.data_type() != precision {
+ return Err(MTLError::Generic(format!(
+ "Weights dtype mismatch: got {:?}, expected {:?}",
+ weights.data_type(),
+ precision
+ )));
+ }
+
+ let weights_buffer: MTLBuffer =
+ unsafe { weights.mtl_buffer() }.to_owned();
+
+ let (bias_add_kernel, biases_buffer) =
+ match parameter_tree.leaf("biases") {
+ Ok(mut biases) => {
+ if biases.shape() != [output_dim] {
+ return Err(MTLError::Generic(format!(
+ "Bias shape mismatch: got {:?}, expected [{:?}]",
+ biases.shape(),
+ output_dim
+ )));
+ }
+ if biases.data_type() != precision {
+ return Err(MTLError::Generic(format!(
+ "Bias dtype mismatch: got {:?}, expected {:?}",
+ biases.data_type(),
+ precision
+ )));
+ }
+ let bias_add_kernel = Some(TensorAddBias::new(
+ mtl_context,
+ KernelDataType::from(precision),
+ )?);
+ let biases_buffer: MTLBuffer =
+ unsafe { biases.mtl_buffer() }.to_owned();
+ (bias_add_kernel, Some(biases_buffer))
+ },
+ Err(_) => (None, None),
+ };
+
+ let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;
+
+ Ok(Self {
+ kernel: RefCell::new(kernel),
+ bias_add_kernel,
+ biases_buffer,
+ weights_buffer,
+ input_dim,
+ output_dim,
+ input_array_id,
+ output_array_id,
+ })
+ }
+}
+
+impl EncodableBlock for FullPrecisionLinear {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let batch_size = state.active_suffix_length();
+
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = MatmulArguments {
+ a: input_buffer,
+ b: &self.weights_buffer,
+ d: output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.input_dim as i32,
+ output_dim: self.output_dim as i32,
+ lda: self.input_dim as i32,
+ ldb: self.input_dim as i32,
+ ldd: self.output_dim as i32,
+ batch_count: 1,
+ };
+
+ self.kernel
+ .borrow_mut()
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
+
+ encoder.end_encoding();
+
+ if let (Some(bias_add), Some(bias_buf)) =
+ (&self.bias_add_kernel, &self.biases_buffer)
+ {
+ let total_len = batch_size * self.output_dim;
+ let retained_cb = root_command_buffer.to_owned();
+ bias_add.encode_into_command_buffer(
+ &output_buffer,
+ bias_buf,
+ &output_buffer,
+ self.output_dim,
+ total_len,
+ &retained_cb,
+ parameters.predicate,
+ );
+ }
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let batch_size = state.active_suffix_length();
+
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let args = MatmulArguments {
+ a: input_buffer,
+ b: &self.weights_buffer,
+ d: output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.input_dim as i32,
+ output_dim: self.output_dim as i32,
+ lda: self.input_dim as i32,
+ ldb: self.input_dim as i32,
+ ldd: self.output_dim as i32,
+ batch_count: 1,
+ };
+
+ self.kernel
+ .borrow_mut()
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
+
+ if let (Some(bias_add), Some(bias_buf)) =
+ (&self.bias_add_kernel, &self.biases_buffer)
+ {
+ let total_len = batch_size * self.output_dim;
+ bias_add.encode_with_encoder(
+ &output_buffer,
+ bias_buf,
+ &output_buffer,
+ self.output_dim,
+ total_len,
+ encoder,
+ parameters.predicate,
+ );
+ }
+ }
+}
+
pub struct QuantizedLinear {
kernel: QuantizedMatmulKernel,
bias_add_kernel: Option<TensorAddBias>,
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 7497d8f95..5577febb6 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -40,7 +40,7 @@ pub use embedding::{
};
pub use encoding_parameters::EncodingParameters;
pub use layer::LayerExecutables;
-pub use linear::QuantizedLinear;
+pub use linear::{FullPrecisionLinear, QuantizedLinear};
pub(crate) use mamba_mixer::MambaMixer;
pub use mlp::MlpBlock;
pub use moe_block::{MoeBlock, SharedMoeWeights};
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 12f439f33..013620942 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -7,8 +7,8 @@ use mpsgraph::{
use objc2::rc::{Retained, autoreleasepool};
use super::{
- EncodableBlock, MlpBlock, MoeBlock, QuantizedEmbeddingLookup,
- QuantizedEmbeddingReadout, QuantizedLinear,
+ EncodableBlock, FullPrecisionLinear, MlpBlock, MoeBlock,
+ QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
@@ -37,7 +37,7 @@ fn make_execution_descriptor() -> Retained<ExecutableExecutionDescriptor> {
pub fn linear_block<const N: usize>(
config: &LinearConfig,
- has_biases: bool,
+ _has_biases: bool,
input_dim: usize,
output_dims: [usize; N],
context: &MTLContext,
@@ -46,31 +46,39 @@ pub fn linear_block<const N: usize>(
output_array_id: ArrayId,
compilation_descriptor: &CompilationDescriptor,
) -> Box<dyn EncodableBlock> {
- if let LinearConfig::Quantized(quant_config) = config {
- let out_sum: usize = output_dims.iter().sum();
- return quantized_linear_block_custom(
- quant_config,
- input_dim,
- out_sum,
- context,
- parameter_tree,
- input_array_id,
- output_array_id,
- )
- .unwrap();
- }
- if let LinearConfig::MLXQuantized(quant_config) = config {
- let out_sum: usize = output_dims.iter().sum();
- return quantized_linear_block_custom(
- quant_config,
- input_dim,
- out_sum,
- context,
- parameter_tree,
- input_array_id,
- output_array_id,
- )
- .unwrap();
+ let out_sum: usize = output_dims.iter().sum();
+ match config {
+ LinearConfig::Quantized(quant_config)
+ | LinearConfig::MLXQuantized(quant_config) => {
+ return quantized_linear_block_custom(
+ quant_config,
+ input_dim,
+ out_sum,
+ context,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )
+ .unwrap();
+ },
+ LinearConfig::FullPrecision {
+ precision,
+ } => {
+ let block = FullPrecisionLinear::new(
+ context,
+ (*precision).into(),
+ input_dim,
+ out_sum,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )
+ .expect("Failed to create full precision linear kernel block");
+ return Box::new(block);
+ },
+ LinearConfig::QLoRA {
+ ..
+ } => {},
}
autoreleasepool(|_| {
@@ -86,7 +94,7 @@ pub fn linear_block<const N: usize>(
config,
input_dim,
output_dims,
- has_biases,
+ true,
&input_placeholder,
parameter_tree,
)
@@ -543,6 +551,10 @@ pub fn readout_block(
precision,
..
} => {
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
+
let weights_shape =
[config.vocab_size as isize, config.model_dim as isize];
let weights_data_type: DataType = precision.into();
@@ -553,6 +565,13 @@ pub fn readout_block(
let weights_transposed =
graph.transpose(&weights_placeholder, &[1, 0], None);
+ let input_shape = [-1 as isize, config.model_dim as isize];
+ let input_data_type: DataType =
+ config.output_norm_config.scale_precision.into();
+ let input_placeholder =
+ placeholder(&graph, &input_shape, input_data_type);
+ let input_shaped_type = shaped_type(&input_shape, input_data_type);
+
let output = embeddings_readout_subgraph(
&graph,
&input_placeholder,
@@ -570,7 +589,7 @@ pub fn readout_block(
&feeds,
&[&output],
None,
- Some(&compilation_descriptor),
+ Some(compilation_descriptor),
);
let block = MPSGraphBlock::new(
@@ -589,6 +608,10 @@ pub fn readout_block(
precision,
..
} => {
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
+ // Use MPSGraph readout path
let weights_shape =
[config.vocab_size as isize, config.model_dim as isize];
let weights_data_type: DataType = precision.into();
@@ -599,6 +622,13 @@ pub fn readout_block(
let weights_transposed =
graph.transpose(&weights_placeholder, &[1, 0], None);
+ let input_shape = [-1 as isize, config.model_dim as isize];
+ let input_data_type: DataType =
+ config.output_norm_config.scale_precision.into();
+ let input_placeholder =
+ placeholder(&graph, &input_shape, input_data_type);
+ let input_shaped_type = shaped_type(&input_shape, input_data_type);
+
let output = embeddings_readout_subgraph(
&graph,
&input_placeholder,
@@ -616,7 +646,7 @@ pub fn readout_block(
&feeds,
&[&output],
None,
- Some(&compilation_descriptor),
+ Some(compilation_descriptor),
);
let block = MPSGraphBlock::new(
@@ -659,7 +689,15 @@ pub fn readout_block(
embedding_quantization_mode: _,
..
} => {
- // Packed U8 weights with shape [vocab_size, model_dim / 2]
+ let graph = Graph::new();
+
+ let input_shape = [-1 as isize, config.model_dim as isize];
+ let input_data_type: DataType =
+ config.output_norm_config.scale_precision.into();
+ let input_placeholder =
+ placeholder(&graph, &input_shape, input_data_type);
+ let input_shaped_type = shaped_type(&input_shape, input_data_type);
+
let weights_shape =
[config.vocab_size as isize, (config.model_dim / 2) as isize];
let weights_data_type = DataType::U8;
@@ -701,7 +739,7 @@ pub fn readout_block(
&feeds,
&[&output],
None,
- Some(&compilation_descriptor),
+ Some(compilation_descriptor),
);
let block = MPSGraphBlock::new(
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
index 4adbc459a..f5513f2af 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
@@ -1,14 +1,20 @@
-use std::mem::size_of;
+use std::{mem::size_of, rc::Rc};
use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef,
ComputePipelineState as MTLComputePipelineState, MTLSize,
};
-use super::super::MTLContext;
+use super::super::{MTLContext, kernel::matmul::MatmulArguments};
use crate::{
- DataType, backends::metal::encodable_block::QuantizedEmbeddingError,
+ DataType,
+ backends::metal::{
+ ForwardPassState, MTLError, encodable_block::QuantizedEmbeddingError,
+ forward_pass::ArrayId,
+ },
config::QuantizationMode,
+ device::array::Array,
+ parameters::ParameterTree,
};
pub struct QuantizedEmbeddingLookupKernel {
@@ -119,3 +125,102 @@ impl QuantizedEmbeddingLookupKernel {
Ok(())
}
}
+
+pub struct FullPrecisionReadoutKernelBlock {
+ kernel: std::cell::RefCell<super::super::kernel::matmul::MatmulKernel>,
+ weights_buffer: MTLBuffer,
+ vocab_size: usize,
+ model_dim: usize,
+}
+
+impl FullPrecisionReadoutKernelBlock {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for full precision readout: {:?}",
+ data_type
+ )));
+ }
+
+ let mut weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
+ MTLError::Generic(format!("Failed to load weights: {:?}", e))
+ })?,
+ };
+
+ if weights.shape() != [vocab_size, model_dim] {
+ return Err(MTLError::Generic(format!(
+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim
+ )));
+ }
+
+ if weights.data_type() != data_type {
+ return Err(MTLError::Generic(format!(
+ "Weights dtype mismatch: got {:?}, expected {:?}",
+ weights.data_type(),
+ data_type
+ )));
+ }
+
+ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
+
+ let kernel = super::super::kernel::matmul::MatmulKernel::new(
+ mtl_context,
+ data_type,
+ false,
+ true,
+ )?;
+
+ Ok(Self {
+ kernel: std::cell::RefCell::new(kernel),
+ weights_buffer,
+ vocab_size,
+ model_dim,
+ })
+ }
+}
+
+impl FullPrecisionReadoutKernelBlock {
+ fn encode_impl(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &metal::ComputeCommandEncoderRef,
+ ) {
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
+ let batch_size = state.active_suffix_length();
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let args = MatmulArguments {
+ a: input_buffer,
+ b: &self.weights_buffer,
+ d: output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.model_dim as i32,
+ output_dim: self.vocab_size as i32,
+ lda: self.model_dim as i32,
+ ldb: self.model_dim as i32, // B is [vocab_size, model_dim], stride is model_dim
+ ldd: self.vocab_size as i32,
+ batch_count: 1,
+ };
+
+ self.kernel
+ .borrow_mut()
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode full precision readout kernel");
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index bcf40230c..0528c0967 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -3,6 +3,7 @@ mod data_type;
pub mod embedding;
pub mod kv_cache_update;
pub mod layer_norm;
+pub mod matmul;
pub mod mask_update;
pub mod media_kernels;
pub mod mlp;
@@ -28,6 +29,7 @@ pub use attention::{
pub use data_type::KernelDataType;
pub use kv_cache_update::KVCacheUpdate;
pub use layer_norm::{LayerNormArguments, LayerNormError, LayerNormKernel};
+pub use matmul::{MatmulArguments, MatmulKernel};
pub use mask_update::MaskUpdateKernel;
pub use moe::{
MoeBlockBasesArguments, MoeCountsOffsetsFusedArguments,