Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

qlora-wip

Aleksei Savinc776095cf07e
diff --git a/Cargo.lock b/Cargo.lock
index 3f50bcd8b..302a46a4b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1312,7 +1312,7 @@ dependencies = [
[[package]]
name = "mpsgraph"
version = "0.1.0"
-source = "git+https://github.com/getmirai/mpsgraph-rs?rev=6a3d803#6a3d803fcb2c102298e2cae49e76a049e388b749"
+source = "git+https://github.com/getmirai/mpsgraph-rs?rev=414c0d2#414c0d23b67d65e525972b8f7486c45f30939289"
dependencies = [
"bindgen",
"bitflags 2.9.1",
diff --git a/uzu/Cargo.toml b/uzu/Cargo.toml
index 89c922ffc..58358b4ca 100644
--- a/uzu/Cargo.toml
+++ b/uzu/Cargo.toml
@@ -12,7 +12,7 @@ serde_json = "1.0.140"
thiserror = "2.0.12"
tokenizers = "0.21.1"
# mpsgraph = { path = "../mpsgraph-rs" }
-mpsgraph = { git = "https://github.com/getmirai/mpsgraph-rs", rev = "6a3d803", package = "mpsgraph" }
+mpsgraph = { git = "https://github.com/getmirai/mpsgraph-rs", rev = "414c0d2", package = "mpsgraph" }
objc2 = "0.6.1"
objc2-foundation = "0.3.1"
metal = "0.32.0"
diff --git a/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/uzu/src/backends/metal/executable_builder/decoder_executables.rs
index 659196a9c..881a00b2a 100644
--- a/uzu/src/backends/metal/executable_builder/decoder_executables.rs
+++ b/uzu/src/backends/metal/executable_builder/decoder_executables.rs
@@ -17,8 +17,8 @@ use crate::{
ArrayId, ForwardPassState, MPSGraphBlock, RopeType,
encodable_with_state::{EncodableWithState, EncodingParameters},
transformer_layer::{
- embedding_with_placeholder_weights_blocks, rms_norm_block,
- rotation_block, rotation_executable,
+ embed_block, readout_block, rms_norm_block, rotation_block,
+ rotation_executable,
},
},
kernel::{RMSNormKernelEncodable, RopeKernelEncodable},
@@ -44,7 +44,6 @@ impl KernelsConfig {
}
}
pub struct DecoderExecutables {
- pub embedding: Option<MPSGraphBlock>,
pub embed: MPSGraphBlock,
pub layers: Box<[LayerExecutables]>,
pub norm: Box<dyn EncodableWithState>,
@@ -62,7 +61,13 @@ impl DecoderExecutables {
attention_executable_provider_config: AttentionExecutableProviderConfig,
kernels_config: KernelsConfig,
) -> Self {
- let embedding_blocks = embedding_with_placeholder_weights_blocks(
+ let embed = embed_block(
+ &decoder_config,
+ &mtl_context,
+ &compilation_config.descriptor_general,
+ );
+
+ let readout = readout_block(
&decoder_config,
&mtl_context,
&compilation_config.descriptor_general,
@@ -176,11 +181,10 @@ impl DecoderExecutables {
};
Self {
- embed: embedding_blocks.embed,
- embedding: embedding_blocks.embedding,
+ embed: embed,
layers: layers.into_boxed_slice(),
norm: norm_block,
- readout: embedding_blocks.readout,
+ readout: readout,
global_rope,
local_rope,
}
diff --git a/uzu/src/backends/metal/forward_pass/model_shape.rs b/uzu/src/backends/metal/forward_pass/model_shape.rs
index ba8bac9e2..d18d3fc16 100644
--- a/uzu/src/backends/metal/forward_pass/model_shape.rs
+++ b/uzu/src/backends/metal/forward_pass/model_shape.rs
@@ -87,6 +87,14 @@ impl ModelShape {
[self.model_dim, self.vocabulary_size]
}
+ pub fn quantized_embeddings_weights_shape(&self) -> [usize; 2] {
+ [self.vocabulary_size, self.model_dim]
+ }
+
+ pub fn quantized_embeddings_scales_shape(&self) -> [usize; 1] {
+ [self.vocabulary_size]
+ }
+
pub fn attention_output_shape(
&self,
suffix_length: usize,
diff --git a/uzu/src/backends/metal/forward_pass/state.rs b/uzu/src/backends/metal/forward_pass/state.rs
index 9d59a588f..e93d23285 100644
--- a/uzu/src/backends/metal/forward_pass/state.rs
+++ b/uzu/src/backends/metal/forward_pass/state.rs
@@ -21,13 +21,19 @@ type ArrayCell = RefCell<MetalArray>;
pub enum EmbeddingsBuffers {
Tied {
/// [vocab_size, model_dim]
- embeddings: ArrayCell,
+ weights: ArrayCell,
},
Untied {
/// [vocab_size, model_dim]
- input: ArrayCell,
+ input_weights: ArrayCell,
/// [vocab_size, model_dim]
- output: ArrayCell,
+ output_weights: ArrayCell,
+ },
+ QuantizedTied {
+ /// [vocab_size, model_dim]
+ weights: ArrayCell,
+ /// [vocab_size]
+ scales: ArrayCell,
},
}
@@ -43,7 +49,7 @@ impl EmbeddingsBuffers {
common: _,
precision: _,
} => Self::Tied {
- embeddings: RefCell::new(context.array_uninitialized(
+ weights: RefCell::new(context.array_uninitialized(
&model_shape.embeddings_input_shape(),
model_shape.activation_data_type(),
)),
@@ -52,17 +58,27 @@ impl EmbeddingsBuffers {
common: _,
precision: _,
} => Self::Untied {
- input: RefCell::new(context.array_uninitialized(
+ input_weights: RefCell::new(context.array_uninitialized(
&model_shape.embeddings_input_shape(),
model_shape.activation_data_type(),
)),
- output: RefCell::new(context.array_uninitialized(
+ output_weights: RefCell::new(context.array_uninitialized(
&model_shape.embeddings_output_shape(),
model_shape.activation_data_type(),
)),
},
- _ => {
- unimplemented!()
+ EmbeddingConfig::QuantizedTied {
+ embedding_quantization_mode,
+ ..
+ } => Self::QuantizedTied {
+ weights: RefCell::new(context.array_uninitialized(
+ &model_shape.quantized_embeddings_weights_shape(),
+ (*embedding_quantization_mode).into(),
+ )),
+ scales: RefCell::new(context.array_uninitialized(
+ &model_shape.quantized_embeddings_scales_shape(),
+ model_shape.activation_data_type(),
+ )),
},
}
}
@@ -75,18 +91,29 @@ impl EmbeddingsBuffers {
let embeddings_tree = parameter_tree.subtree("embedding").unwrap();
match self {
EmbeddingsBuffers::Tied {
- embeddings,
+ weights,
} => {
- let embeddings_view =
- embeddings_tree.leaf("token_embeddings").unwrap();
- embeddings.borrow_mut().copy_from_array(&embeddings_view);
+ let embeddings_view = embeddings_tree.leaf("weights").unwrap();
+ weights.borrow_mut().copy_from_array(&embeddings_view);
},
EmbeddingsBuffers::Untied {
- input,
- output,
+ input_weights,
+ output_weights,
+ } => {
+ let mapping = vec![
+ ("input_weights", input_weights),
+ ("output_weights", output_weights),
+ ];
+ for (name, buffer) in mapping {
+ let view = embeddings_tree.leaf(name).unwrap();
+ buffer.borrow_mut().copy_from_array(&view);
+ }
+ },
+ EmbeddingsBuffers::QuantizedTied {
+ weights,
+ scales,
} => {
- let mapping =
- vec![("input_weights", input), ("output_weights", output)];
+ let mapping = vec![("weights", weights), ("scales", scales)];
for (name, buffer) in mapping {
let view = embeddings_tree.leaf(name).unwrap();
buffer.borrow_mut().copy_from_array(&view);
@@ -463,26 +490,43 @@ impl ForwardPassState {
ArrayId::AttentionSums => self.aux_buffers.attention_sums.clone(),
ArrayId::AttentionMaxs => self.aux_buffers.attention_maxs.clone(),
- ArrayId::EmbeddingsInput => {
+ ArrayId::EmbeddingsInputWeights => {
match &self.shared_buffers.borrow().embeddings {
EmbeddingsBuffers::Tied {
- embeddings,
- } => embeddings.clone(),
+ weights,
+ } => weights.clone(),
EmbeddingsBuffers::Untied {
- input,
- output: _,
- } => input.clone(),
+ input_weights,
+ ..
+ } => input_weights.clone(),
+ EmbeddingsBuffers::QuantizedTied {
+ weights,
+ ..
+ } => weights.clone(),
}
},
- ArrayId::EmbeddingsOutput => {
+ ArrayId::EmbeddingsOutputWeights => {
match &self.shared_buffers.borrow().embeddings {
EmbeddingsBuffers::Tied {
- embeddings,
- } => embeddings.clone(),
+ weights,
+ } => weights.clone(),
EmbeddingsBuffers::Untied {
- input: _,
- output,
- } => output.clone(),
+ output_weights,
+ ..
+ } => output_weights.clone(),
+ EmbeddingsBuffers::QuantizedTied {
+ weights,
+ ..
+ } => weights.clone(),
+ }
+ },
+ ArrayId::EmbeddingsScales => {
+ match &self.shared_buffers.borrow().embeddings {
+ EmbeddingsBuffers::QuantizedTied {
+ scales,
+ ..
+ } => scales.clone(),
+ _ => panic!("Expected EmbeddingsBuffers::QuantizedTied"),
}
},
ArrayId::RopeCosines(rope_type) => match rope_type {
@@ -605,8 +649,9 @@ pub enum ArrayId {
AttentionSums,
AttentionMaxs,
- EmbeddingsInput,
- EmbeddingsOutput,
+ EmbeddingsInputWeights,
+ EmbeddingsOutputWeights,
+ EmbeddingsScales,
RopeCosines(RopeType),
RopeSines(RopeType),
}
diff --git a/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/uzu/src/backends/metal/forward_pass/transformer_layer.rs
index a26eee2d6..469937775 100644
--- a/uzu/src/backends/metal/forward_pass/transformer_layer.rs
+++ b/uzu/src/backends/metal/forward_pass/transformer_layer.rs
@@ -2,7 +2,7 @@ use std::{collections::HashMap, rc::Rc};
use mpsgraph::{
CompilationDescriptor, Device as MPSDevice, Executable,
- ExecutableExecutionDescriptor, Graph, ShapedType, Tensor,
+ ExecutableExecutionDescriptor, Graph, GraphTensorShapeOps,
};
use objc2::rc::{Retained, autoreleasepool};
@@ -10,11 +10,8 @@ use super::{
super::{
MTLContext,
graph::{
- EmbeddingParams, attention_subgraph, embed_callable_subgraph,
- embed_placeholder_weights_subgraph, embed_subgraph,
- embedding_params, linear_subgraph, mlp_subgraph, placeholder,
- readout_callable_subgraph, readout_placeholder_weights_subgraph,
- readout_subgraph, rms_norm_subgraph, shaped_type,
+ attention_subgraph, linear_subgraph, mlp_subgraph, placeholder,
+ rms_norm_subgraph, shaped_type,
},
},
io_arrays::IOArrays,
@@ -23,7 +20,10 @@ use super::{
};
use crate::{
DataType,
- backends::metal::graph::rotation_subgraph,
+ backends::metal::graph::{
+ embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
+ embeddings_readout_subgraph, rotation_subgraph,
+ },
config::{
DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig, RMSNormConfig,
RoPEConfig,
@@ -354,379 +354,340 @@ pub fn attention_block(
})
}
-pub struct EmbeddingBlocks {
- pub embedding: Option<MPSGraphBlock>,
- pub embed: MPSGraphBlock,
- pub readout: MPSGraphBlock,
-}
-
-pub fn embedding_callable_blocks(
+pub fn embed_block(
config: &DecoderConfig,
context: &MTLContext,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
compilation_descriptor: &CompilationDescriptor,
-) -> EmbeddingBlocks {
- autoreleasepool(|_| {
- let embedding_graph = Graph::new();
- let embedding_params = embedding_params(
- &embedding_graph,
- &config.embedding_config,
- config.vocab_size,
- config.model_dim,
- parameter_tree,
- )
- .unwrap();
-
- match embedding_params {
- EmbeddingParams::Tied(embeddings) => {
- let embedding_callable_name: &str = "embedding";
- let embedding_output = embeddings;
- let embedding_executable = embedding_graph.compile(
- &MPSDevice::with_device(&context.device),
- &HashMap::<&Tensor, &ShapedType>::new(),
- &[&embedding_output],
- Some(compilation_descriptor),
- );
-
- let compilation_desctiptor_with_embedding_callable =
- CompilationDescriptor::new();
- compilation_desctiptor_with_embedding_callable.add_callable(
- embedding_callable_name,
- &embedding_executable,
- );
-
- let embed_graph = Graph::new();
- let embed_input_shape = [-1 as isize];
- let embed_input_data_type = DataType::U64;
- let embed_input_placeholder = placeholder(
- &embed_graph,
- &embed_input_shape,
- embed_input_data_type,
- );
- let retained_embed_input_shaped_type =
- shaped_type(&embed_input_shape, embed_input_data_type);
- let embed_output = embed_callable_subgraph(
- &embed_graph,
- &config.embedding_config,
- embedding_callable_name,
- [config.vocab_size as isize, config.model_dim as isize],
- &embed_input_placeholder,
- )
- .unwrap();
-
- let embed_feeds = HashMap::from([(
- &*embed_input_placeholder,
- &*retained_embed_input_shaped_type,
- )]);
-
- let embed_executable = embed_graph.compile(
- &MPSDevice::with_device(&context.device),
- &embed_feeds,
- &[&embed_output],
- Some(&compilation_desctiptor_with_embedding_callable),
- );
-
- let readout_graph = Graph::new();
- let readout_input_shape =
- [-1 as isize, config.model_dim as isize];
- let readout_input = placeholder(
- &readout_graph,
- &readout_input_shape,
- config.output_norm_config.scale_precision.into(),
- );
- let retained_readout_input_shaped_type = shaped_type(
- &readout_input_shape,
- config.output_norm_config.scale_precision.into(),
- );
- let readout_output = readout_callable_subgraph(
- &readout_graph,
- &config.embedding_config,
- embedding_callable_name,
- [config.vocab_size as isize, config.model_dim as isize],
- &readout_input,
- )
- .unwrap();
-
- let readout_feeds = HashMap::from([(
- &*readout_input,
- &*retained_readout_input_shaped_type,
- )]);
-
- let readout_executable = readout_graph.compile(
- &MPSDevice::with_device(&context.device),
- &readout_feeds,
- &[&readout_output],
- Some(&compilation_desctiptor_with_embedding_callable),
- );
-
- let embedding_block = MPSGraphBlock::new(
- embedding_executable,
- make_execution_descriptor(),
- IOArrays::new(Box::new([]), Box::new([])),
- );
- let embed_block = MPSGraphBlock::new(
- embed_executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::TokenIds].into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- ),
- );
- let readout_block = MPSGraphBlock::new(
- readout_executable,
- make_execution_descriptor(),
- IOArrays::new(
- Box::new([ArrayId::Main]),
- Box::new([ArrayId::Logits]),
- ),
- );
- EmbeddingBlocks {
- embedding: Some(embedding_block),
- embed: embed_block,
- readout: readout_block,
- }
- },
- EmbeddingParams::Untied {
- input_weights: _,
- output_weights: _,
- } => {
- unimplemented!()
- },
- }
- })
-}
-
-pub fn embedding_blocks(
- config: &DecoderConfig,
- context: &MTLContext,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- compilation_descriptor: &CompilationDescriptor,
-) -> EmbeddingBlocks {
- autoreleasepool(|_| {
- let embed_graph = Graph::new();
- let embed_input_shape = [-1 as isize];
- let embed_input_data_type = DataType::U64;
- let embed_input_placeholder = placeholder(
- &embed_graph,
- &embed_input_shape,
- embed_input_data_type,
- );
- let retained_embed_input_shaped_type =
- shaped_type(&embed_input_shape, embed_input_data_type);
- let embed_output = embed_subgraph(
- &embed_graph,
- &config.embedding_config,
- config.vocab_size,
- config.model_dim,
- &embed_input_placeholder,
- parameter_tree,
- )
- .unwrap();
-
- let embed_feeds = HashMap::from([(
- &*embed_input_placeholder,
- &*retained_embed_input_shaped_type,
- )]);
-
- let embed_executable = embed_graph.compile(
- &MPSDevice::with_device(&context.device),
- &embed_feeds,
- &[&embed_output],
- Some(compilation_descriptor),
- );
-
- let readout_graph = Graph::new();
- let readout_input_shape = [-1 as isize, config.model_dim as isize];
- let readout_input = placeholder(
- &readout_graph,
- &readout_input_shape,
- config.output_norm_config.scale_precision.into(),
- );
- let retained_readout_input_shaped_type = shaped_type(
- &readout_input_shape,
- config.output_norm_config.scale_precision.into(),
- );
- let readout_output = readout_subgraph(
- &readout_graph,
- &config.embedding_config,
- config.vocab_size,
- config.model_dim,
- &readout_input,
- parameter_tree,
- )
- .unwrap();
-
- let readout_feeds = HashMap::from([(
- &*readout_input,
- &*retained_readout_input_shaped_type,
- )]);
-
- let readout_executable = readout_graph.compile(
- &MPSDevice::with_device(&context.device),
- &readout_feeds,
- &[&readout_output],
- Some(compilation_descriptor),
- );
+) -> MPSGraphBlock {
+ let graph = Graph::new();
+
+ let input_shape = [-1 as isize];
+ let input_data_type = DataType::U64;
+ let input_placeholder = placeholder(&graph, &input_shape, input_data_type);
+ let input_shaped_type = shaped_type(&input_shape, input_data_type);
+
+ match config.embedding_config {
+ EmbeddingConfig::Tied {
+ precision,
+ ..
+ } => {
+ let weights_shape =
+ [config.vocab_size as isize, config.model_dim as isize];
+ let weights_data_type: DataType = precision.into();
+ let weights_shaped_type =
+ shaped_type(&weights_shape, weights_data_type);
+ let weights_placeholder =
+ placeholder(&graph, &weights_shape, weights_data_type);
+
+ let output = embeddings_embed_subgraph(
+ &graph,
+ &config.embedding_config,
+ &input_placeholder,
+ &weights_placeholder,
+ )
+ .unwrap();
+
+ let feeds = HashMap::from([
+ (&*input_placeholder, &*input_shaped_type),
+ (&*weights_placeholder, &*weights_shaped_type),
+ ]);
+
+ let executable = graph.compile(
+ &MPSDevice::with_device(&context.device),
+ &feeds,
+ &[&output],
+ Some(compilation_descriptor),
+ );
+
+ let block = MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ IOArrays::new(
+ vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]
+ .into_boxed_slice(),
+ vec![ArrayId::Main].into_boxed_slice(),
+ ),
+ );
+
+ block
+ },
+ EmbeddingConfig::Untied {
+ precision,
+ ..
+ } => {
+ let weights_shape =
+ [config.vocab_size as isize, config.model_dim as isize];
+ let weights_data_type: DataType = precision.into();
+ let weights_shaped_type =
+ shaped_type(&weights_shape, weights_data_type);
+ let weights_placeholder =
+ placeholder(&graph, &weights_shape, weights_data_type);
+
+ let output = embeddings_embed_subgraph(
+ &graph,
+ &config.embedding_config,
+ &input_placeholder,
+ &weights_placeholder,
+ )
+ .unwrap();
+
+ let feeds = HashMap::from([
+ (&*input_placeholder, &*input_shaped_type),
+ (&*weights_placeholder, &*weights_shaped_type),
+ ]);
+
+ let executable = graph.compile(
+ &MPSDevice::with_device(&context.device),
+ &feeds,
+ &[&output],
+ Some(compilation_descriptor),
+ );
+
+ let block = MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ IOArrays::new(
+ vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]
+ .into_boxed_slice(),
+ vec![ArrayId::Main].into_boxed_slice(),
+ ),
+ );
+
+ block
+ },
+ EmbeddingConfig::QuantizedTied {
+ embedding_quantization_mode,
+ ..
+ } => {
+ let weights_shape =
+ [config.vocab_size as isize, config.model_dim as isize];
+ let weights_data_type: DataType =
+ embedding_quantization_mode.into();
+ let weights_shaped_type =
+ shaped_type(&weights_shape, weights_data_type);
+ let weights_placeholder =
+ placeholder(&graph, &weights_shape, weights_data_type);
+
+ let scales_shape = [config.vocab_size as isize];
+ let scales_data_type: DataType =
+ config.output_norm_config.scale_precision.into();
+ let scales_shaped_type =
+ shaped_type(&scales_shape, scales_data_type);
+ let scales_placeholder =
+ placeholder(&graph, &scales_shape, scales_data_type);
+
+ let weights = embeddings_dequantize_weights_subgraph(
+ &graph,
+ &weights_placeholder,
+ &scales_placeholder,
+ )
+ .unwrap();
+
+ let output = embeddings_embed_subgraph(
+ &graph,
+ &config.embedding_config,
+ &input_placeholder,
+ &weights,
+ )
+ .unwrap();
+
+ let feeds = HashMap::from([
+ (&*input_placeholder, &*input_shaped_type),
+ (&*weights_placeholder, &*weights_shaped_type),
+ (&*scales_placeholder, &*scales_shaped_type),
+ ]);
+
+ let executable = graph.compile(
+ &MPSDevice::with_device(&context.device),
+ &feeds,
+ &[&output],
+ Some(compilation_descriptor),
+ );
+
+ let block = MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ IOArrays::new(
+ vec![
+ ArrayId::TokenIds,
+ ArrayId::EmbeddingsInputWeights,
+ ArrayId::EmbeddingsScales,
+ ]
+ .into_boxed_slice(),
+ vec![ArrayId::Main].into_boxed_slice(),
+ ),
+ );
- let embed_block = MPSGraphBlock::new(
- embed_executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::TokenIds].into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- ),
- );
- let readout_block = MPSGraphBlock::new(
- readout_executable,
- make_execution_descriptor(),
- IOArrays::new(
- Box::new([ArrayId::Main]),
- Box::new([ArrayId::Logits]),
- ),
- );
- EmbeddingBlocks {
- embedding: None,
- embed: embed_block,
- readout: readout_block,
- }
- })
+ block
+ },
+ }
}
-pub fn embedding_with_placeholder_weights_blocks(
+pub fn readout_block(
config: &DecoderConfig,
context: &MTLContext,
compilation_descriptor: &CompilationDescriptor,
-) -> EmbeddingBlocks {
- autoreleasepool(|_| {
- let embed_weights_shape =
- [config.vocab_size as isize, config.model_dim as isize];
- let embed_weights_data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let embed_weights_shaped_type =
- shaped_type(&embed_weights_shape, embed_weights_data_type);
-
- let embed_graph = Graph::new();
- let embed_input_shape = [-1 as isize];
- let embed_input_data_type = DataType::U64;
- let embed_input_placeholder = placeholder(
- &embed_graph,
- &embed_input_shape,
- embed_input_data_type,
- );
- let embed_input_shaped_type =
- shaped_type(&embed_input_shape, embed_input_data_type);
- let embed_weights_placeholder = placeholder(
- &embed_graph,
- &embed_weights_shape,
- embed_weights_data_type,
- );
- let embed_output = embed_placeholder_weights_subgraph(
- &embed_graph,
- &config.embedding_config,
- &embed_input_placeholder,
- &embed_weights_placeholder,
- )
- .unwrap();
-
- let embed_feeds = HashMap::from([
- (&*embed_input_placeholder, &*embed_input_shaped_type),
- (&*embed_weights_placeholder, &*embed_weights_shaped_type),
- ]);
-
- let embed_executable = embed_graph.compile(
- &MPSDevice::with_device(&context.device),
- &embed_feeds,
- &[&embed_output],
- Some(compilation_descriptor),
- );
-
- let embed_block = MPSGraphBlock::new(
- embed_executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInput]
+) -> MPSGraphBlock {
+ let graph = Graph::new();
+
+ let input_shape = [-1 as isize, config.model_dim as isize];
+ let input_data_type: DataType =
+ config.output_norm_config.scale_precision.into();
+ let input_placeholder = placeholder(&graph, &input_shape, input_data_type);
+ let input_shaped_type = shaped_type(&input_shape, input_data_type);
+
+ match config.embedding_config {
+ EmbeddingConfig::Tied {
+ precision,
+ ..
+ } => {
+ let weights_shape =
+ [config.vocab_size as isize, config.model_dim as isize];
+ let weights_data_type: DataType = precision.into();
+ let weights_shaped_type =
+ shaped_type(&weights_shape, weights_data_type);
+ let weights_placeholder =
+ placeholder(&graph, &weights_shape, weights_data_type);
+
+ let weights_transposed =
+ graph.transpose(&weights_placeholder, &[1, 0], None);
+ let output = embeddings_readout_subgraph(
+ &graph,
+ &input_placeholder,
+ &weights_transposed,
+ )
+ .unwrap();
+
+ let feeds = HashMap::from([
+ (&*input_placeholder, &*input_shaped_type),
+ (&*weights_placeholder, &*weights_shaped_type),
+ ]);
+
+ let executable = graph.compile(
+ &MPSDevice::with_device(&context.device),
+ &feeds,
+ &[&output],
+ Some(compilation_descriptor),
+ );
+
+ let block = MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ IOArrays::new(
+ vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]
+ .into_boxed_slice(),
+ vec![ArrayId::Logits].into_boxed_slice(),
+ ),
+ );
+
+ block
+ },
+ EmbeddingConfig::Untied {
+ precision,
+ ..
+ } => {
+ let weights_shape =
+ [config.model_dim as isize, config.vocab_size as isize];
+ let weights_data_type: DataType = precision.into();
+ let weights_shaped_type =
+ shaped_type(&weights_shape, weights_data_type);
+ let weights_placeholder =
+ placeholder(&graph, &weights_shape, weights_data_type);
+
+ let output = embeddings_readout_subgraph(
+ &graph,
+ &input_placeholder,
+ &weights_placeholder,
+ )
+ .unwrap();
+
+ let feeds = HashMap::from([
+ (&*input_placeholder, &*input_shaped_type),
+ (&*weights_placeholder, &*weights_shaped_type),
+ ]);
+
+ let executable = graph.compile(
+ &MPSDevice::with_device(&context.device),
+ &feeds,
+ &[&output],
+ Some(compilation_descriptor),
+ );
+
+ let block = MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ IOArrays::new(
+ vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]
+ .into_boxed_slice(),
+ vec![ArrayId::Logits].into_boxed_slice(),
+ ),
+ );
+
+ block
+ },
+ EmbeddingConfig::QuantizedTied {
+ embedding_quantization_mode,
+ ..
+ } => {
+ let weights_shape =
+ [config.vocab_size as isize, config.model_dim as isize];
+ let weights_data_type: DataType =
+ embedding_quantization_mode.into();
+ let weights_shaped_type =
+ shaped_type(&weights_shape, weights_data_type);
+ let weights_placeholder =
+ placeholder(&graph, &weights_shape, weights_data_type);
+
+ let scales_shape = [config.vocab_size as isize];
+ let scales_data_type: DataType =
+ config.output_norm_config.scale_precision.into();
+ let scales_shaped_type =
+ shaped_type(&scales_shape, scales_data_type);
+ let scales_placeholder =
+ placeholder(&graph, &scales_shape, scales_data_type);
+
+ let weights = embeddings_dequantize_weights_subgraph(
+ &graph,
+ &weights_placeholder,
+ &scales_placeholder,
+ )
+ .unwrap();
+ let weights_transposed = graph.transpose(&weights, &[1, 0], None);
+ let output = embeddings_readout_subgraph(
+ &graph,
+ &input_placeholder,
+ &weights_transposed,
+ )
+ .unwrap();
+
+ let feeds = HashMap::from([
+ (&*input_placeholder, &*input_shaped_type),
+ (&*weights_placeholder, &*weights_shaped_type),
+ (&*scales_placeholder, &*scales_shaped_type),
+ ]);
+
+ let executable = graph.compile(
+ &MPSDevice::with_device(&context.device),
+ &feeds,
+ &[&output],
+ Some(compilation_descriptor),
+ );
+
+ let block = MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ IOArrays::new(
+ vec![
+ ArrayId::Main,
+ ArrayId::EmbeddingsOutputWeights,
+ ArrayId::EmbeddingsScales,
+ ]
.into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- ),
- );
-
- let readout_weights_shape: [isize; 2];
- let readout_weights_data_type: DataType;
- let readout_transpose_weights: bool;
- match config.embedding_config {
- EmbeddingConfig::Tied {
- common: _,
- precision,
- } => {
- readout_weights_shape =
- [config.vocab_size as isize, config.model_dim as isize];
- readout_weights_data_type = precision.into();
- readout_transpose_weights = true;
- },
- EmbeddingConfig::Untied {
- common: _,
- precision,
- } => {
- readout_weights_shape =
- [config.model_dim as isize, config.vocab_size as isize];
- readout_weights_data_type = precision.into();
- readout_transpose_weights = false;
- },
- _ => {
- unimplemented!()
- },
- }
- let readout_weights_shaped_type =
- shaped_type(&readout_weights_shape, readout_weights_data_type);
-
- let readout_graph = Graph::new();
- let readout_input_shape = [-1 as isize, config.model_dim as isize];
- let readout_input = placeholder(
- &readout_graph,
- &readout_input_shape,
- config.output_norm_config.scale_precision.into(),
- );
- let readout_input_shaped_type = shaped_type(
- &readout_input_shape,
- config.output_norm_config.scale_precision.into(),
- );
- let readout_weights_placeholder = placeholder(
- &readout_graph,
- &readout_weights_shape,
- readout_weights_data_type,
- );
- let readout_output = readout_placeholder_weights_subgraph(
- &readout_graph,
- &readout_input,
- &readout_weights_placeholder,
- readout_transpose_weights,
- )
- .unwrap();
+ vec![ArrayId::Logits].into_boxed_slice(),
+ ),
+ );
- let readout_feeds = HashMap::from([
- (&*readout_input, &*readout_input_shaped_type),
- (&*readout_weights_placeholder, &*readout_weights_shaped_type),
- ]);
-
- let readout_executable = readout_graph.compile(
- &MPSDevice::with_device(&context.device),
- &readout_feeds,
- &[&readout_output],
- Some(compilation_descriptor),
- );
-
- let readout_block = MPSGraphBlock::new(
- readout_executable,
- make_execution_descriptor(),
- IOArrays::new(
- Box::new([ArrayId::Main, ArrayId::EmbeddingsOutput]),
- Box::new([ArrayId::Logits]),
- ),
- );
- EmbeddingBlocks {
- embedding: None,
- embed: embed_block,
- readout: readout_block,
- }
- })
+ block
+ },
+ }
}
diff --git a/uzu/src/backends/metal/graph/embedding.rs b/uzu/src/backends/metal/graph/embedding.rs
index 8350eff10..d6e6e8529 100644
--- a/uzu/src/backends/metal/graph/embedding.rs
+++ b/uzu/src/backends/metal/graph/embedding.rs
@@ -1,119 +1,30 @@
-#![allow(dead_code)]
-use std::rc::Rc;
-
-use mpsgraph::{Graph, GraphCallOps, GraphGatherOps, GraphMatrixOps, GraphTensorShapeOps, Tensor};
-use objc2::rc::Retained;
-
-use super::{
- super::MTLContext,
- GraphConstructionError,
- common::{load_constant, shaped_type},
+use mpsgraph::{
+ Graph, GraphGatherOps, GraphMatrixOps, GraphQuantizationOps, Tensor,
};
-use crate::{config::EmbeddingConfig, parameters::ParameterTree};
-
-pub enum EmbeddingParams {
- Tied(Retained<Tensor>),
- Untied {
- input_weights: Retained<Tensor>,
- output_weights: Retained<Tensor>,
- },
-}
-
-pub fn embedding_params(
- graph: &Graph,
- config: &EmbeddingConfig,
- vocabulary_size: usize,
- model_dim: usize,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<EmbeddingParams, GraphConstructionError> {
- match config {
- EmbeddingConfig::Tied {
- common: _,
- precision,
- } => {
- let weights = load_constant(
- graph,
- &parameter_tree.subtree("embedding").unwrap(),
- "token_embeddings",
- &[vocabulary_size, model_dim],
- (*precision).into(),
- )?;
- Ok(EmbeddingParams::Tied(weights))
- },
- _ => {
- unimplemented!()
- },
- }
-}
+use objc2::rc::Retained;
-pub fn embed_callable_subgraph(
- graph: &Graph,
- config: &EmbeddingConfig,
- embeddings_callable_name: &str,
- embedding_dims: [isize; 2],
- input: &Tensor,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- match config {
- EmbeddingConfig::Tied {
- common: _,
- precision,
- } => {
- let embeddings_shaped_type =
- shaped_type(&embedding_dims, (*precision).into());
- if let Some(embeddings) = graph
- .call(
- embeddings_callable_name,
- &[],
- &[&embeddings_shaped_type],
- None,
- )
- .first()
- {
- let result = graph.gather(embeddings, input, 0, 0, None);
- Ok(result)
- } else {
- return Err(GraphConstructionError::CallableResultError {
- name: embeddings_callable_name.to_string(),
- });
- }
- },
- _ => {
- unimplemented!()
- },
- }
-}
+use super::GraphConstructionError;
+use crate::config::EmbeddingConfig;
-pub fn embed_subgraph(
- graph: &Graph,
- config: &EmbeddingConfig,
- vocabulary_size: usize,
- model_dim: usize,
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
+pub fn embeddings_dequantize_weights_subgraph(
+ graph: &Retained<Graph>,
+ weights: &Retained<Tensor>,
+ scales: &Retained<Tensor>,
) -> Result<Retained<Tensor>, GraphConstructionError> {
- match config {
- EmbeddingConfig::Tied {
- common: _,
- precision,
- } => {
- let embeddings = load_constant(
- graph,
- &parameter_tree.subtree("embedding").unwrap(),
- "token_embeddings",
- &[vocabulary_size, model_dim],
- (*precision).into(),
- )
- .unwrap();
- let result = graph.gather(&embeddings, input, 0, 0, None);
- Ok(result)
- },
- _ => {
- unimplemented!()
- },
- }
-}
-
-pub fn embed_placeholder_weights_subgraph(
+ let result = graph
+ .dequantize_with_scale_tensor_zero_point_and_axis(
+ &weights,
+ &scales,
+ 0.0,
+ scales.data_type(),
+ 0,
+ None,
+ )
+ .unwrap();
+ Ok(result)
+}
+
+pub fn embeddings_embed_subgraph(
graph: &Retained<Graph>,
config: &EmbeddingConfig,
input: &Retained<Tensor>,
@@ -130,95 +41,11 @@ pub fn embed_placeholder_weights_subgraph(
}
}
-pub fn readout_callable_subgraph(
- graph: &Graph,
- config: &EmbeddingConfig,
- embeddings_callable_name: &str,
- embedding_dims: [isize; 2],
- input: &Tensor,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- match config {
- EmbeddingConfig::Tied {
- common: _,
- precision,
- } => {
- let embeddings_shaped_type =
- shaped_type(&embedding_dims, (*precision).into());
- if let Some(embeddings) = graph
- .call(
- embeddings_callable_name,
- &[],
- &[&embeddings_shaped_type],
- None,
- )
- .first()
- {
- let embeddings_transposed =
- graph.transpose(embeddings, &[1, 0], None);
- let result = graph.matmul(
- input,
- &embeddings_transposed,
- None,
- );
- Ok(result)
- } else {
- return Err(GraphConstructionError::CallableResultError {
- name: embeddings_callable_name.to_string(),
- });
- }
- },
- _ => {
- unimplemented!()
- },
- }
-}
-
-pub fn readout_subgraph(
- graph: &Graph,
- config: &EmbeddingConfig,
- vocabulary_size: usize,
- model_dim: usize,
- input: &Tensor,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- match config {
- EmbeddingConfig::Tied {
- common: _,
- precision,
- } => {
- let embeddings = load_constant(
- graph,
- &parameter_tree.subtree("embedding").unwrap(),
- "token_embeddings",
- &[vocabulary_size, model_dim],
- (*precision).into(),
- )
- .unwrap();
- let embeddings_transposed =
- graph.transpose(&embeddings, &[1, 0], None);
- let result =
- graph.matmul(input, &embeddings_transposed, None);
- Ok(result)
- },
- _ => {
- unimplemented!()
- },
- }
-}
-
-pub fn readout_placeholder_weights_subgraph(
+pub fn embeddings_readout_subgraph(
graph: &Retained<Graph>,
input: &Retained<Tensor>,
weights: &Retained<Tensor>,
- transpose_weights: bool,
) -> Result<Retained<Tensor>, GraphConstructionError> {
- if transpose_weights {
- let embeddings_transposed = graph.transpose(weights, &[1, 0], None);
- let result =
- graph.matmul(input, &embeddings_transposed, None);
- Ok(result)
- } else {
- let result = graph.matmul(input, weights, None);
- Ok(result)
- }
+ let result = graph.matmul(input, weights, None);
+ Ok(result)
}
diff --git a/uzu/src/backends/metal/graph/linear.rs b/uzu/src/backends/metal/graph/linear.rs
index 23630c811..49af38f39 100644
--- a/uzu/src/backends/metal/graph/linear.rs
+++ b/uzu/src/backends/metal/graph/linear.rs
@@ -8,129 +8,325 @@ use objc2::rc::Retained;
use super::{super::MTLContext, GraphConstructionError, load_constant};
use crate::{
DataType,
- config::{ConfigDataType, LinearConfig},
+ config::{
+ ConfigDataType, LinearConfig, QuantizationConfig, QuantizationMode,
+ },
parameters::ParameterTree,
};
-pub fn linear_subgraph<const N: usize>(
+fn full_precision_weights_subgraph<const N: usize>(
graph: &Graph,
- config: &LinearConfig,
+ precision: DataType,
input_dim: usize,
output_dims: [usize; N],
- has_biases: bool,
- input: &Tensor,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Result<Retained<Tensor>, GraphConstructionError> {
- match config {
- LinearConfig::FullPrecision {
- precision,
- } => {
- let output_dim_sum: usize = output_dims.iter().sum();
- let weights = load_constant(
+ let output_dim_sum: usize = output_dims.iter().sum();
+ load_constant(
+ graph,
+ parameter_tree,
+ "weights",
+ &[input_dim, output_dim_sum],
+ precision,
+ )
+}
+
+fn quantized_weights_subgraph<const N: usize>(
+ graph: &Graph,
+ config: &QuantizationConfig,
+ input_dim: usize,
+ output_dims: [usize; N],
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Retained<Tensor>, GraphConstructionError> {
+ if config.weight_quantization_mode != QuantizationMode::UInt4 {
+ unimplemented!("8-bit quantization is not yet implemented.");
+ }
+ let output_dim_sum: usize = output_dims.iter().sum();
+
+ let weights = load_constant(
+ graph,
+ parameter_tree,
+ "weights",
+ &[input_dim, output_dim_sum],
+ DataType::U4,
+ )?;
+
+ let scales = load_constant(
+ graph,
+ parameter_tree,
+ "scales",
+ &[input_dim, output_dim_sum / config.group_size],
+ config.activation_precision.into(),
+ )?;
+
+ let zero_points = load_constant(
+ graph,
+ parameter_tree,
+ "zero_points",
+ &[input_dim, output_dim_sum / config.group_size],
+ DataType::U4,
+ )?;
+
+ Ok(graph
+ .dequantize_with_scale_tensor_and_zero_point_tensor(
+ &weights,
+ &scales,
+ &zero_points,
+ <ConfigDataType as Into<DataType>>::into(
+ config.activation_precision,
+ )
+ .into(),
+ None,
+ )
+ .unwrap())
+}
+
+fn biases_subgraph<const N: usize>(
+ graph: &Graph,
+ precision: DataType,
+ output_dims: [usize; N],
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Retained<Tensor>, GraphConstructionError> {
+ let output_dim_sum: usize = output_dims.iter().sum();
+ load_constant(graph, parameter_tree, "biases", &[output_dim_sum], precision)
+}
+
+struct LoraWeights {
+ down_weights: Retained<Tensor>,
+ up_weights: Box<[Retained<Tensor>]>,
+}
+
+fn lora_weights_subgraph<const N: usize>(
+ graph: &Graph,
+ precision: DataType,
+ input_dim: usize,
+ output_dims: [usize; N],
+ lora_rank: usize,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<LoraWeights, GraphConstructionError> {
+ let down_weights = load_constant(
+ graph,
+ parameter_tree,
+ "down_weights",
+ &[input_dim, lora_rank * N],
+ precision,
+ )?;
+ let up_weights = output_dims
+ .iter()
+ .enumerate()
+ .map(|(i, output_dim)| {
+ load_constant(
graph,
parameter_tree,
- "weights",
- &[input_dim, output_dim_sum],
- (*precision).into(),
- )?;
+ &format!("up_weights.{}", i),
+ &[lora_rank, *output_dim],
+ precision,
+ )
+ })
+ .collect::<Result<Box<[_]>, _>>()?;
+ Ok(LoraWeights {
+ down_weights,
+ up_weights: up_weights,
+ })
+}
- let matmul = graph.transpose(
+fn full_precision_matmul_subgraph<const N: usize>(
+ graph: &Graph,
+ precision: DataType,
+ input_dim: usize,
+ output_dims: [usize; N],
+ input: &Tensor,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Retained<Tensor>, GraphConstructionError> {
+ let weights = full_precision_weights_subgraph::<N>(
+ graph,
+ precision,
+ input_dim,
+ output_dims,
+ parameter_tree,
+ )?;
+ Ok(graph.transpose(
+ &graph.matmul(
+ &graph.transpose(&weights, &[1, 0], None),
+ &graph.transpose(input, &[1, 0], None),
+ None,
+ ),
+ &[1, 0],
+ None,
+ ))
+}
+
+fn quantized_matmul_subgraph<const N: usize>(
+ graph: &Graph,
+ config: &QuantizationConfig,
+ input_dim: usize,
+ output_dims: [usize; N],
+ input: &Tensor,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Retained<Tensor>, GraphConstructionError> {
+ let weights = quantized_weights_subgraph::<N>(
+ graph,
+ config,
+ input_dim,
+ output_dims,
+ parameter_tree,
+ )?;
+ Ok(graph.matmul(input, &weights, None))
+}
+
+fn lora_subgraph<const N: usize>(
+ graph: &Graph,
+ precision: DataType,
+ lora_rank: usize,
+ input_dim: usize,
+ output_dims: [usize; N],
+ input: &Tensor,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Retained<Tensor>, GraphConstructionError> {
+ let lora_weights = lora_weights_subgraph(
+ graph,
+ precision,
+ input_dim,
+ output_dims,
+ lora_rank,
+ parameter_tree,
+ )?;
+
+ let lora_inner = graph.transpose(
+ &graph.matmul(
+ &graph.transpose(&lora_weights.down_weights, &[1, 0], None),
+ &graph.transpose(input, &[1, 0], None),
+ None,
+ ),
+ &[1, 0],
+ None,
+ );
+
+ let lora_inner_splits =
+ graph.split(&lora_inner, output_dims.len(), 1, None);
+ assert_eq!(lora_inner_splits.len(), lora_weights.up_weights.len());
+
+ let lora_outputs = lora_inner_splits
+ .iter()
+ .zip(lora_weights.up_weights.iter())
+ .map(|(lora_inner_split, up_weights)| {
+ graph.transpose(
&graph.matmul(
- &graph.transpose(&weights, &[1, 0], None),
- &graph.transpose(input, &[1, 0], None),
+ &graph.transpose(&up_weights, &[1, 0], None),
+ &graph.transpose(&lora_inner_split, &[1, 0], None),
None,
),
&[1, 0],
None,
- );
-
- if has_biases {
- let biases = load_constant(
- graph,
- parameter_tree,
- "biases",
- &[output_dim_sum],
- (*precision).into(),
- )?;
-
- let result = graph.add(&matmul, &biases, None);
-
- Ok(result)
- } else {
- Ok(matmul)
- }
- },
- LinearConfig::Quantized(quantization_config) => {
- let output_dim_sum: usize = output_dims.iter().sum();
- let group_size = quantization_config.group_size;
- let activation_precision = quantization_config.activation_precision;
+ )
+ })
+ .collect::<Box<[Retained<Tensor>]>>();
- let weights = load_constant(
- graph,
- parameter_tree,
- "weights",
- &[output_dim_sum, input_dim],
- DataType::U4,
- )?;
+ Ok(graph.concat_tensors(
+ &lora_outputs
+ .iter()
+ .map(|tensor| &**tensor)
+ .collect::<Box<[&Tensor]>>(),
+ 1,
+ None,
+ ))
+}
- let scales = load_constant(
- graph,
- parameter_tree,
- "scales",
- &[output_dim_sum, input_dim / group_size],
- activation_precision.into(),
- )?;
+fn qlora_matmul_subgraph<const N: usize>(
+ graph: &Graph,
+ lora_rank: usize,
+ lora_scale: f32,
+ config: &QuantizationConfig,
+ input_dim: usize,
+ output_dims: [usize; N],
+ input: &Tensor,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Retained<Tensor>, GraphConstructionError> {
+ let quantized_matmul_result = quantized_matmul_subgraph::<N>(
+ graph,
+ config,
+ input_dim,
+ output_dims,
+ input,
+ parameter_tree,
+ )?;
+ let lora_result = lora_subgraph::<N>(
+ graph,
+ config.activation_precision.into(),
+ lora_rank,
+ input_dim,
+ output_dims,
+ input,
+ parameter_tree,
+ )?;
+ let scaled_lora_result = graph.multiply(
+ &lora_result,
+ &graph.constant_with_scalar(
+ lora_scale as f64,
+ <ConfigDataType as Into<DataType>>::into(
+ config.activation_precision,
+ )
+ .into(),
+ ),
+ None,
+ );
+ Ok(graph.add(&quantized_matmul_result, &scaled_lora_result, None))
+}
- let zero_points = load_constant(
+pub fn linear_subgraph<const N: usize>(
+ graph: &Graph,
+ config: &LinearConfig,
+ input_dim: usize,
+ output_dims: [usize; N],
+ has_biases: bool,
+ input: &Tensor,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Retained<Tensor>, GraphConstructionError> {
+ let matmul_result = match config {
+ LinearConfig::FullPrecision {
+ precision,
+ } => full_precision_matmul_subgraph(
+ graph,
+ (*precision).into(),
+ input_dim,
+ output_dims,
+ input,
+ parameter_tree,
+ )?,
+ LinearConfig::Quantized(quantization_config) => {
+ quantized_matmul_subgraph(
graph,
+ quantization_config,
+ input_dim,
+ output_dims,
+ input,
parameter_tree,
- "zero_points",
- &[output_dim_sum, input_dim / group_size],
- DataType::U4,
- )?;
-
- let dequantized_weights = graph
- .dequantize_with_scale_tensor_and_zero_point_tensor(
- &weights,
- &scales,
- &zero_points,
- <ConfigDataType as Into<DataType>>::into(
- activation_precision,
- )
- .into(),
- None,
- )
- .unwrap();
-
- let matmul = graph.matmul(
- &input,
- &graph.transpose(&dequantized_weights, &[1, 0], None),
- None,
- );
-
- if has_biases {
- let biases = load_constant(
- graph,
- parameter_tree,
- "biases",
- &[output_dim_sum],
- activation_precision.into(),
- )?;
-
- let result = graph.add(&matmul, &biases, None);
-
- Ok(result)
- } else {
- Ok(matmul)
- }
+ )?
},
LinearConfig::QLoRA {
- ..
- } => {
- // QLoRA linear layer implementation
- unimplemented!(
- "QLoRA linear layer implementation not yet available"
- )
- },
+ quantization: quantization_config,
+ lora_rank,
+ lora_scale,
+ } => qlora_matmul_subgraph(
+ graph,
+ *lora_rank,
+ *lora_scale,
+ quantization_config,
+ input_dim,
+ output_dims,
+ input,
+ parameter_tree,
+ )?,
+ };
+ if has_biases {
+ let biases = biases_subgraph(
+ graph,
+ config.activation_precision().into(),
+ output_dims,
+ parameter_tree,
+ )?;
+ Ok(graph.add(&matmul_result, &biases, None))
+ } else {
+ Ok(matmul_result)
}
}
diff --git a/uzu/src/backends/metal/graph/mod.rs b/uzu/src/backends/metal/graph/mod.rs
index fae91049b..27a626eb1 100644
--- a/uzu/src/backends/metal/graph/mod.rs
+++ b/uzu/src/backends/metal/graph/mod.rs
@@ -11,9 +11,7 @@ mod rms_norm;
pub use rms_norm::rms_norm_subgraph;
mod embedding;
pub use embedding::{
- EmbeddingParams, embed_callable_subgraph,
- embed_placeholder_weights_subgraph, embed_subgraph, embedding_params,
- readout_callable_subgraph, readout_placeholder_weights_subgraph,
- readout_subgraph,
+ embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
+ embeddings_readout_subgraph,
};
mod rope;
diff --git a/uzu/src/config/common.rs b/uzu/src/config/common.rs
index aa7d54783..f2f1896ce 100644
--- a/uzu/src/config/common.rs
+++ b/uzu/src/config/common.rs
@@ -46,6 +46,16 @@ pub enum QuantizationMode {
UInt8,
}
+impl Into<DataType> for QuantizationMode {
+ fn into(self) -> DataType {
+ match self {
+ QuantizationMode::UInt4 => DataType::U4,
+ QuantizationMode::Int8 => DataType::I8,
+ QuantizationMode::UInt8 => DataType::U8,
+ }
+ }
+}
+
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Copy, Clone)]
#[serde(rename_all = "lowercase")]
pub enum Activation {
diff --git a/uzu/tests/parameters_loader_test.rs b/uzu/tests/parameters_loader_test.rs
index a56647d97..36a10606b 100644
--- a/uzu/tests/parameters_loader_test.rs
+++ b/uzu/tests/parameters_loader_test.rs
@@ -14,12 +14,12 @@ fn test_parameter_loader_basic() {
let loader = ParameterLoader::new(&file, &context).expect("create loader");
let embeddings =
- loader.get("embedding.token_embeddings").expect("load embeddings");
+ loader.get("embedding.weights").expect("weights embeddings");
let view = embeddings.as_view::<f16>().unwrap();
assert!(is_close!(view[[5usize, 3usize]], f16::from_f32(-0.01819)));
// tree API check
let subtree = loader.tree().subtree("embedding").unwrap();
- let same = subtree.leaf("token_embeddings").unwrap();
+ let same = subtree.leaf("weights").unwrap();
assert_eq!(view, same.as_view::<f16>().unwrap());
}