A high-performance inference engine for AI models
qlora-wip
Aleksei Savinc776095cf07e
diff --git a/Cargo.lock b/Cargo.lockindex 3f50bcd8b..302a46a4b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1312,7 +1312,7 @@ dependencies = [[[package]]name = "mpsgraph"version = "0.1.0"-source = "git+https://github.com/getmirai/mpsgraph-rs?rev=6a3d803#6a3d803fcb2c102298e2cae49e76a049e388b749"+source = "git+https://github.com/getmirai/mpsgraph-rs?rev=414c0d2#414c0d23b67d65e525972b8f7486c45f30939289"dependencies = ["bindgen","bitflags 2.9.1",diff --git a/uzu/Cargo.toml b/uzu/Cargo.tomlindex 89c922ffc..58358b4ca 100644--- a/uzu/Cargo.toml+++ b/uzu/Cargo.toml@@ -12,7 +12,7 @@ serde_json = "1.0.140"thiserror = "2.0.12"tokenizers = "0.21.1"# mpsgraph = { path = "../mpsgraph-rs" }-mpsgraph = { git = "https://github.com/getmirai/mpsgraph-rs", rev = "6a3d803", package = "mpsgraph" }+mpsgraph = { git = "https://github.com/getmirai/mpsgraph-rs", rev = "414c0d2", package = "mpsgraph" }objc2 = "0.6.1"objc2-foundation = "0.3.1"metal = "0.32.0"diff --git a/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/uzu/src/backends/metal/executable_builder/decoder_executables.rsindex 659196a9c..881a00b2a 100644--- a/uzu/src/backends/metal/executable_builder/decoder_executables.rs+++ b/uzu/src/backends/metal/executable_builder/decoder_executables.rs@@ -17,8 +17,8 @@ use crate::{ArrayId, ForwardPassState, MPSGraphBlock, RopeType,encodable_with_state::{EncodableWithState, EncodingParameters},transformer_layer::{- embedding_with_placeholder_weights_blocks, rms_norm_block,- rotation_block, rotation_executable,+ embed_block, readout_block, rms_norm_block, rotation_block,+ rotation_executable,},},kernel::{RMSNormKernelEncodable, RopeKernelEncodable},@@ -44,7 +44,6 @@ impl KernelsConfig {}}pub struct DecoderExecutables {- pub embedding: Option<MPSGraphBlock>,pub embed: MPSGraphBlock,pub layers: Box<[LayerExecutables]>,pub norm: Box<dyn EncodableWithState>,@@ -62,7 +61,13 @@ impl DecoderExecutables {attention_executable_provider_config: AttentionExecutableProviderConfig,kernels_config: KernelsConfig,) -> Self {- let embedding_blocks = embedding_with_placeholder_weights_blocks(+ let embed = embed_block(+ &decoder_config,+ &mtl_context,+ &compilation_config.descriptor_general,+ );++ let readout = readout_block(&decoder_config,&mtl_context,&compilation_config.descriptor_general,@@ -176,11 +181,10 @@ impl DecoderExecutables {};Self {- embed: embedding_blocks.embed,- embedding: embedding_blocks.embedding,+ embed: embed,layers: layers.into_boxed_slice(),norm: norm_block,- readout: embedding_blocks.readout,+ readout: readout,global_rope,local_rope,}diff --git a/uzu/src/backends/metal/forward_pass/model_shape.rs b/uzu/src/backends/metal/forward_pass/model_shape.rsindex ba8bac9e2..d18d3fc16 100644--- a/uzu/src/backends/metal/forward_pass/model_shape.rs+++ b/uzu/src/backends/metal/forward_pass/model_shape.rs@@ -87,6 +87,14 @@ impl ModelShape {[self.model_dim, self.vocabulary_size]}+ pub fn quantized_embeddings_weights_shape(&self) -> [usize; 2] {+ [self.vocabulary_size, self.model_dim]+ }++ pub fn quantized_embeddings_scales_shape(&self) -> [usize; 1] {+ [self.vocabulary_size]+ }+pub fn attention_output_shape(&self,suffix_length: usize,diff --git a/uzu/src/backends/metal/forward_pass/state.rs b/uzu/src/backends/metal/forward_pass/state.rsindex 9d59a588f..e93d23285 100644--- a/uzu/src/backends/metal/forward_pass/state.rs+++ b/uzu/src/backends/metal/forward_pass/state.rs@@ -21,13 +21,19 @@ type ArrayCell = RefCell<MetalArray>;pub enum EmbeddingsBuffers {Tied {/// [vocab_size, model_dim]- embeddings: ArrayCell,+ weights: ArrayCell,},Untied {/// [vocab_size, model_dim]- input: ArrayCell,+ input_weights: ArrayCell,/// [vocab_size, model_dim]- output: ArrayCell,+ output_weights: ArrayCell,+ },+ QuantizedTied {+ /// [vocab_size, model_dim]+ weights: ArrayCell,+ /// [vocab_size]+ scales: ArrayCell,},}@@ -43,7 +49,7 @@ impl EmbeddingsBuffers {common: _,precision: _,} => Self::Tied {- embeddings: RefCell::new(context.array_uninitialized(+ weights: RefCell::new(context.array_uninitialized(&model_shape.embeddings_input_shape(),model_shape.activation_data_type(),)),@@ -52,17 +58,27 @@ impl EmbeddingsBuffers {common: _,precision: _,} => Self::Untied {- input: RefCell::new(context.array_uninitialized(+ input_weights: RefCell::new(context.array_uninitialized(&model_shape.embeddings_input_shape(),model_shape.activation_data_type(),)),- output: RefCell::new(context.array_uninitialized(+ output_weights: RefCell::new(context.array_uninitialized(&model_shape.embeddings_output_shape(),model_shape.activation_data_type(),)),},- _ => {- unimplemented!()+ EmbeddingConfig::QuantizedTied {+ embedding_quantization_mode,+ ..+ } => Self::QuantizedTied {+ weights: RefCell::new(context.array_uninitialized(+ &model_shape.quantized_embeddings_weights_shape(),+ (*embedding_quantization_mode).into(),+ )),+ scales: RefCell::new(context.array_uninitialized(+ &model_shape.quantized_embeddings_scales_shape(),+ model_shape.activation_data_type(),+ )),},}}@@ -75,18 +91,29 @@ impl EmbeddingsBuffers {let embeddings_tree = parameter_tree.subtree("embedding").unwrap();match self {EmbeddingsBuffers::Tied {- embeddings,+ weights,} => {- let embeddings_view =- embeddings_tree.leaf("token_embeddings").unwrap();- embeddings.borrow_mut().copy_from_array(&embeddings_view);+ let embeddings_view = embeddings_tree.leaf("weights").unwrap();+ weights.borrow_mut().copy_from_array(&embeddings_view);},EmbeddingsBuffers::Untied {- input,- output,+ input_weights,+ output_weights,+ } => {+ let mapping = vec![+ ("input_weights", input_weights),+ ("output_weights", output_weights),+ ];+ for (name, buffer) in mapping {+ let view = embeddings_tree.leaf(name).unwrap();+ buffer.borrow_mut().copy_from_array(&view);+ }+ },+ EmbeddingsBuffers::QuantizedTied {+ weights,+ scales,} => {- let mapping =- vec![("input_weights", input), ("output_weights", output)];+ let mapping = vec![("weights", weights), ("scales", scales)];for (name, buffer) in mapping {let view = embeddings_tree.leaf(name).unwrap();buffer.borrow_mut().copy_from_array(&view);@@ -463,26 +490,43 @@ impl ForwardPassState {ArrayId::AttentionSums => self.aux_buffers.attention_sums.clone(),ArrayId::AttentionMaxs => self.aux_buffers.attention_maxs.clone(),- ArrayId::EmbeddingsInput => {+ ArrayId::EmbeddingsInputWeights => {match &self.shared_buffers.borrow().embeddings {EmbeddingsBuffers::Tied {- embeddings,- } => embeddings.clone(),+ weights,+ } => weights.clone(),EmbeddingsBuffers::Untied {- input,- output: _,- } => input.clone(),+ input_weights,+ ..+ } => input_weights.clone(),+ EmbeddingsBuffers::QuantizedTied {+ weights,+ ..+ } => weights.clone(),}},- ArrayId::EmbeddingsOutput => {+ ArrayId::EmbeddingsOutputWeights => {match &self.shared_buffers.borrow().embeddings {EmbeddingsBuffers::Tied {- embeddings,- } => embeddings.clone(),+ weights,+ } => weights.clone(),EmbeddingsBuffers::Untied {- input: _,- output,- } => output.clone(),+ output_weights,+ ..+ } => output_weights.clone(),+ EmbeddingsBuffers::QuantizedTied {+ weights,+ ..+ } => weights.clone(),+ }+ },+ ArrayId::EmbeddingsScales => {+ match &self.shared_buffers.borrow().embeddings {+ EmbeddingsBuffers::QuantizedTied {+ scales,+ ..+ } => scales.clone(),+ _ => panic!("Expected EmbeddingsBuffers::QuantizedTied"),}},ArrayId::RopeCosines(rope_type) => match rope_type {@@ -605,8 +649,9 @@ pub enum ArrayId {AttentionSums,AttentionMaxs,- EmbeddingsInput,- EmbeddingsOutput,+ EmbeddingsInputWeights,+ EmbeddingsOutputWeights,+ EmbeddingsScales,RopeCosines(RopeType),RopeSines(RopeType),}diff --git a/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/uzu/src/backends/metal/forward_pass/transformer_layer.rsindex a26eee2d6..469937775 100644--- a/uzu/src/backends/metal/forward_pass/transformer_layer.rs+++ b/uzu/src/backends/metal/forward_pass/transformer_layer.rs@@ -2,7 +2,7 @@ use std::{collections::HashMap, rc::Rc};use mpsgraph::{CompilationDescriptor, Device as MPSDevice, Executable,- ExecutableExecutionDescriptor, Graph, ShapedType, Tensor,+ ExecutableExecutionDescriptor, Graph, GraphTensorShapeOps,};use objc2::rc::{Retained, autoreleasepool};@@ -10,11 +10,8 @@ use super::{super::{MTLContext,graph::{- EmbeddingParams, attention_subgraph, embed_callable_subgraph,- embed_placeholder_weights_subgraph, embed_subgraph,- embedding_params, linear_subgraph, mlp_subgraph, placeholder,- readout_callable_subgraph, readout_placeholder_weights_subgraph,- readout_subgraph, rms_norm_subgraph, shaped_type,+ attention_subgraph, linear_subgraph, mlp_subgraph, placeholder,+ rms_norm_subgraph, shaped_type,},},io_arrays::IOArrays,@@ -23,7 +20,10 @@ use super::{};use crate::{DataType,- backends::metal::graph::rotation_subgraph,+ backends::metal::graph::{+ embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,+ embeddings_readout_subgraph, rotation_subgraph,+ },config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig, RMSNormConfig,RoPEConfig,@@ -354,379 +354,340 @@ pub fn attention_block(})}-pub struct EmbeddingBlocks {- pub embedding: Option<MPSGraphBlock>,- pub embed: MPSGraphBlock,- pub readout: MPSGraphBlock,-}--pub fn embedding_callable_blocks(+pub fn embed_block(config: &DecoderConfig,context: &MTLContext,- parameter_tree: &ParameterTree<Rc<MTLContext>>,compilation_descriptor: &CompilationDescriptor,-) -> EmbeddingBlocks {- autoreleasepool(|_| {- let embedding_graph = Graph::new();- let embedding_params = embedding_params(- &embedding_graph,- &config.embedding_config,- config.vocab_size,- config.model_dim,- parameter_tree,- )- .unwrap();-- match embedding_params {- EmbeddingParams::Tied(embeddings) => {- let embedding_callable_name: &str = "embedding";- let embedding_output = embeddings;- let embedding_executable = embedding_graph.compile(- &MPSDevice::with_device(&context.device),- &HashMap::<&Tensor, &ShapedType>::new(),- &[&embedding_output],- Some(compilation_descriptor),- );-- let compilation_desctiptor_with_embedding_callable =- CompilationDescriptor::new();- compilation_desctiptor_with_embedding_callable.add_callable(- embedding_callable_name,- &embedding_executable,- );-- let embed_graph = Graph::new();- let embed_input_shape = [-1 as isize];- let embed_input_data_type = DataType::U64;- let embed_input_placeholder = placeholder(- &embed_graph,- &embed_input_shape,- embed_input_data_type,- );- let retained_embed_input_shaped_type =- shaped_type(&embed_input_shape, embed_input_data_type);- let embed_output = embed_callable_subgraph(- &embed_graph,- &config.embedding_config,- embedding_callable_name,- [config.vocab_size as isize, config.model_dim as isize],- &embed_input_placeholder,- )- .unwrap();-- let embed_feeds = HashMap::from([(- &*embed_input_placeholder,- &*retained_embed_input_shaped_type,- )]);-- let embed_executable = embed_graph.compile(- &MPSDevice::with_device(&context.device),- &embed_feeds,- &[&embed_output],- Some(&compilation_desctiptor_with_embedding_callable),- );-- let readout_graph = Graph::new();- let readout_input_shape =- [-1 as isize, config.model_dim as isize];- let readout_input = placeholder(- &readout_graph,- &readout_input_shape,- config.output_norm_config.scale_precision.into(),- );- let retained_readout_input_shaped_type = shaped_type(- &readout_input_shape,- config.output_norm_config.scale_precision.into(),- );- let readout_output = readout_callable_subgraph(- &readout_graph,- &config.embedding_config,- embedding_callable_name,- [config.vocab_size as isize, config.model_dim as isize],- &readout_input,- )- .unwrap();-- let readout_feeds = HashMap::from([(- &*readout_input,- &*retained_readout_input_shaped_type,- )]);-- let readout_executable = readout_graph.compile(- &MPSDevice::with_device(&context.device),- &readout_feeds,- &[&readout_output],- Some(&compilation_desctiptor_with_embedding_callable),- );-- let embedding_block = MPSGraphBlock::new(- embedding_executable,- make_execution_descriptor(),- IOArrays::new(Box::new([]), Box::new([])),- );- let embed_block = MPSGraphBlock::new(- embed_executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::TokenIds].into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- ),- );- let readout_block = MPSGraphBlock::new(- readout_executable,- make_execution_descriptor(),- IOArrays::new(- Box::new([ArrayId::Main]),- Box::new([ArrayId::Logits]),- ),- );- EmbeddingBlocks {- embedding: Some(embedding_block),- embed: embed_block,- readout: readout_block,- }- },- EmbeddingParams::Untied {- input_weights: _,- output_weights: _,- } => {- unimplemented!()- },- }- })-}--pub fn embedding_blocks(- config: &DecoderConfig,- context: &MTLContext,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- compilation_descriptor: &CompilationDescriptor,-) -> EmbeddingBlocks {- autoreleasepool(|_| {- let embed_graph = Graph::new();- let embed_input_shape = [-1 as isize];- let embed_input_data_type = DataType::U64;- let embed_input_placeholder = placeholder(- &embed_graph,- &embed_input_shape,- embed_input_data_type,- );- let retained_embed_input_shaped_type =- shaped_type(&embed_input_shape, embed_input_data_type);- let embed_output = embed_subgraph(- &embed_graph,- &config.embedding_config,- config.vocab_size,- config.model_dim,- &embed_input_placeholder,- parameter_tree,- )- .unwrap();-- let embed_feeds = HashMap::from([(- &*embed_input_placeholder,- &*retained_embed_input_shaped_type,- )]);-- let embed_executable = embed_graph.compile(- &MPSDevice::with_device(&context.device),- &embed_feeds,- &[&embed_output],- Some(compilation_descriptor),- );-- let readout_graph = Graph::new();- let readout_input_shape = [-1 as isize, config.model_dim as isize];- let readout_input = placeholder(- &readout_graph,- &readout_input_shape,- config.output_norm_config.scale_precision.into(),- );- let retained_readout_input_shaped_type = shaped_type(- &readout_input_shape,- config.output_norm_config.scale_precision.into(),- );- let readout_output = readout_subgraph(- &readout_graph,- &config.embedding_config,- config.vocab_size,- config.model_dim,- &readout_input,- parameter_tree,- )- .unwrap();-- let readout_feeds = HashMap::from([(- &*readout_input,- &*retained_readout_input_shaped_type,- )]);-- let readout_executable = readout_graph.compile(- &MPSDevice::with_device(&context.device),- &readout_feeds,- &[&readout_output],- Some(compilation_descriptor),- );+) -> MPSGraphBlock {+ let graph = Graph::new();++ let input_shape = [-1 as isize];+ let input_data_type = DataType::U64;+ let input_placeholder = placeholder(&graph, &input_shape, input_data_type);+ let input_shaped_type = shaped_type(&input_shape, input_data_type);++ match config.embedding_config {+ EmbeddingConfig::Tied {+ precision,+ ..+ } => {+ let weights_shape =+ [config.vocab_size as isize, config.model_dim as isize];+ let weights_data_type: DataType = precision.into();+ let weights_shaped_type =+ shaped_type(&weights_shape, weights_data_type);+ let weights_placeholder =+ placeholder(&graph, &weights_shape, weights_data_type);++ let output = embeddings_embed_subgraph(+ &graph,+ &config.embedding_config,+ &input_placeholder,+ &weights_placeholder,+ )+ .unwrap();++ let feeds = HashMap::from([+ (&*input_placeholder, &*input_shaped_type),+ (&*weights_placeholder, &*weights_shaped_type),+ ]);++ let executable = graph.compile(+ &MPSDevice::with_device(&context.device),+ &feeds,+ &[&output],+ Some(compilation_descriptor),+ );++ let block = MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ IOArrays::new(+ vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]+ .into_boxed_slice(),+ vec![ArrayId::Main].into_boxed_slice(),+ ),+ );++ block+ },+ EmbeddingConfig::Untied {+ precision,+ ..+ } => {+ let weights_shape =+ [config.vocab_size as isize, config.model_dim as isize];+ let weights_data_type: DataType = precision.into();+ let weights_shaped_type =+ shaped_type(&weights_shape, weights_data_type);+ let weights_placeholder =+ placeholder(&graph, &weights_shape, weights_data_type);++ let output = embeddings_embed_subgraph(+ &graph,+ &config.embedding_config,+ &input_placeholder,+ &weights_placeholder,+ )+ .unwrap();++ let feeds = HashMap::from([+ (&*input_placeholder, &*input_shaped_type),+ (&*weights_placeholder, &*weights_shaped_type),+ ]);++ let executable = graph.compile(+ &MPSDevice::with_device(&context.device),+ &feeds,+ &[&output],+ Some(compilation_descriptor),+ );++ let block = MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ IOArrays::new(+ vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]+ .into_boxed_slice(),+ vec![ArrayId::Main].into_boxed_slice(),+ ),+ );++ block+ },+ EmbeddingConfig::QuantizedTied {+ embedding_quantization_mode,+ ..+ } => {+ let weights_shape =+ [config.vocab_size as isize, config.model_dim as isize];+ let weights_data_type: DataType =+ embedding_quantization_mode.into();+ let weights_shaped_type =+ shaped_type(&weights_shape, weights_data_type);+ let weights_placeholder =+ placeholder(&graph, &weights_shape, weights_data_type);++ let scales_shape = [config.vocab_size as isize];+ let scales_data_type: DataType =+ config.output_norm_config.scale_precision.into();+ let scales_shaped_type =+ shaped_type(&scales_shape, scales_data_type);+ let scales_placeholder =+ placeholder(&graph, &scales_shape, scales_data_type);++ let weights = embeddings_dequantize_weights_subgraph(+ &graph,+ &weights_placeholder,+ &scales_placeholder,+ )+ .unwrap();++ let output = embeddings_embed_subgraph(+ &graph,+ &config.embedding_config,+ &input_placeholder,+ &weights,+ )+ .unwrap();++ let feeds = HashMap::from([+ (&*input_placeholder, &*input_shaped_type),+ (&*weights_placeholder, &*weights_shaped_type),+ (&*scales_placeholder, &*scales_shaped_type),+ ]);++ let executable = graph.compile(+ &MPSDevice::with_device(&context.device),+ &feeds,+ &[&output],+ Some(compilation_descriptor),+ );++ let block = MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ IOArrays::new(+ vec![+ ArrayId::TokenIds,+ ArrayId::EmbeddingsInputWeights,+ ArrayId::EmbeddingsScales,+ ]+ .into_boxed_slice(),+ vec![ArrayId::Main].into_boxed_slice(),+ ),+ );- let embed_block = MPSGraphBlock::new(- embed_executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::TokenIds].into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- ),- );- let readout_block = MPSGraphBlock::new(- readout_executable,- make_execution_descriptor(),- IOArrays::new(- Box::new([ArrayId::Main]),- Box::new([ArrayId::Logits]),- ),- );- EmbeddingBlocks {- embedding: None,- embed: embed_block,- readout: readout_block,- }- })+ block+ },+ }}-pub fn embedding_with_placeholder_weights_blocks(+pub fn readout_block(config: &DecoderConfig,context: &MTLContext,compilation_descriptor: &CompilationDescriptor,-) -> EmbeddingBlocks {- autoreleasepool(|_| {- let embed_weights_shape =- [config.vocab_size as isize, config.model_dim as isize];- let embed_weights_data_type: DataType =- config.output_norm_config.scale_precision.into();- let embed_weights_shaped_type =- shaped_type(&embed_weights_shape, embed_weights_data_type);-- let embed_graph = Graph::new();- let embed_input_shape = [-1 as isize];- let embed_input_data_type = DataType::U64;- let embed_input_placeholder = placeholder(- &embed_graph,- &embed_input_shape,- embed_input_data_type,- );- let embed_input_shaped_type =- shaped_type(&embed_input_shape, embed_input_data_type);- let embed_weights_placeholder = placeholder(- &embed_graph,- &embed_weights_shape,- embed_weights_data_type,- );- let embed_output = embed_placeholder_weights_subgraph(- &embed_graph,- &config.embedding_config,- &embed_input_placeholder,- &embed_weights_placeholder,- )- .unwrap();-- let embed_feeds = HashMap::from([- (&*embed_input_placeholder, &*embed_input_shaped_type),- (&*embed_weights_placeholder, &*embed_weights_shaped_type),- ]);-- let embed_executable = embed_graph.compile(- &MPSDevice::with_device(&context.device),- &embed_feeds,- &[&embed_output],- Some(compilation_descriptor),- );-- let embed_block = MPSGraphBlock::new(- embed_executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInput]+) -> MPSGraphBlock {+ let graph = Graph::new();++ let input_shape = [-1 as isize, config.model_dim as isize];+ let input_data_type: DataType =+ config.output_norm_config.scale_precision.into();+ let input_placeholder = placeholder(&graph, &input_shape, input_data_type);+ let input_shaped_type = shaped_type(&input_shape, input_data_type);++ match config.embedding_config {+ EmbeddingConfig::Tied {+ precision,+ ..+ } => {+ let weights_shape =+ [config.vocab_size as isize, config.model_dim as isize];+ let weights_data_type: DataType = precision.into();+ let weights_shaped_type =+ shaped_type(&weights_shape, weights_data_type);+ let weights_placeholder =+ placeholder(&graph, &weights_shape, weights_data_type);++ let weights_transposed =+ graph.transpose(&weights_placeholder, &[1, 0], None);+ let output = embeddings_readout_subgraph(+ &graph,+ &input_placeholder,+ &weights_transposed,+ )+ .unwrap();++ let feeds = HashMap::from([+ (&*input_placeholder, &*input_shaped_type),+ (&*weights_placeholder, &*weights_shaped_type),+ ]);++ let executable = graph.compile(+ &MPSDevice::with_device(&context.device),+ &feeds,+ &[&output],+ Some(compilation_descriptor),+ );++ let block = MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ IOArrays::new(+ vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]+ .into_boxed_slice(),+ vec![ArrayId::Logits].into_boxed_slice(),+ ),+ );++ block+ },+ EmbeddingConfig::Untied {+ precision,+ ..+ } => {+ let weights_shape =+ [config.model_dim as isize, config.vocab_size as isize];+ let weights_data_type: DataType = precision.into();+ let weights_shaped_type =+ shaped_type(&weights_shape, weights_data_type);+ let weights_placeholder =+ placeholder(&graph, &weights_shape, weights_data_type);++ let output = embeddings_readout_subgraph(+ &graph,+ &input_placeholder,+ &weights_placeholder,+ )+ .unwrap();++ let feeds = HashMap::from([+ (&*input_placeholder, &*input_shaped_type),+ (&*weights_placeholder, &*weights_shaped_type),+ ]);++ let executable = graph.compile(+ &MPSDevice::with_device(&context.device),+ &feeds,+ &[&output],+ Some(compilation_descriptor),+ );++ let block = MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ IOArrays::new(+ vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]+ .into_boxed_slice(),+ vec![ArrayId::Logits].into_boxed_slice(),+ ),+ );++ block+ },+ EmbeddingConfig::QuantizedTied {+ embedding_quantization_mode,+ ..+ } => {+ let weights_shape =+ [config.vocab_size as isize, config.model_dim as isize];+ let weights_data_type: DataType =+ embedding_quantization_mode.into();+ let weights_shaped_type =+ shaped_type(&weights_shape, weights_data_type);+ let weights_placeholder =+ placeholder(&graph, &weights_shape, weights_data_type);++ let scales_shape = [config.vocab_size as isize];+ let scales_data_type: DataType =+ config.output_norm_config.scale_precision.into();+ let scales_shaped_type =+ shaped_type(&scales_shape, scales_data_type);+ let scales_placeholder =+ placeholder(&graph, &scales_shape, scales_data_type);++ let weights = embeddings_dequantize_weights_subgraph(+ &graph,+ &weights_placeholder,+ &scales_placeholder,+ )+ .unwrap();+ let weights_transposed = graph.transpose(&weights, &[1, 0], None);+ let output = embeddings_readout_subgraph(+ &graph,+ &input_placeholder,+ &weights_transposed,+ )+ .unwrap();++ let feeds = HashMap::from([+ (&*input_placeholder, &*input_shaped_type),+ (&*weights_placeholder, &*weights_shaped_type),+ (&*scales_placeholder, &*scales_shaped_type),+ ]);++ let executable = graph.compile(+ &MPSDevice::with_device(&context.device),+ &feeds,+ &[&output],+ Some(compilation_descriptor),+ );++ let block = MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ IOArrays::new(+ vec![+ ArrayId::Main,+ ArrayId::EmbeddingsOutputWeights,+ ArrayId::EmbeddingsScales,+ ].into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- ),- );-- let readout_weights_shape: [isize; 2];- let readout_weights_data_type: DataType;- let readout_transpose_weights: bool;- match config.embedding_config {- EmbeddingConfig::Tied {- common: _,- precision,- } => {- readout_weights_shape =- [config.vocab_size as isize, config.model_dim as isize];- readout_weights_data_type = precision.into();- readout_transpose_weights = true;- },- EmbeddingConfig::Untied {- common: _,- precision,- } => {- readout_weights_shape =- [config.model_dim as isize, config.vocab_size as isize];- readout_weights_data_type = precision.into();- readout_transpose_weights = false;- },- _ => {- unimplemented!()- },- }- let readout_weights_shaped_type =- shaped_type(&readout_weights_shape, readout_weights_data_type);-- let readout_graph = Graph::new();- let readout_input_shape = [-1 as isize, config.model_dim as isize];- let readout_input = placeholder(- &readout_graph,- &readout_input_shape,- config.output_norm_config.scale_precision.into(),- );- let readout_input_shaped_type = shaped_type(- &readout_input_shape,- config.output_norm_config.scale_precision.into(),- );- let readout_weights_placeholder = placeholder(- &readout_graph,- &readout_weights_shape,- readout_weights_data_type,- );- let readout_output = readout_placeholder_weights_subgraph(- &readout_graph,- &readout_input,- &readout_weights_placeholder,- readout_transpose_weights,- )- .unwrap();+ vec![ArrayId::Logits].into_boxed_slice(),+ ),+ );- let readout_feeds = HashMap::from([- (&*readout_input, &*readout_input_shaped_type),- (&*readout_weights_placeholder, &*readout_weights_shaped_type),- ]);-- let readout_executable = readout_graph.compile(- &MPSDevice::with_device(&context.device),- &readout_feeds,- &[&readout_output],- Some(compilation_descriptor),- );-- let readout_block = MPSGraphBlock::new(- readout_executable,- make_execution_descriptor(),- IOArrays::new(- Box::new([ArrayId::Main, ArrayId::EmbeddingsOutput]),- Box::new([ArrayId::Logits]),- ),- );- EmbeddingBlocks {- embedding: None,- embed: embed_block,- readout: readout_block,- }- })+ block+ },+ }}diff --git a/uzu/src/backends/metal/graph/embedding.rs b/uzu/src/backends/metal/graph/embedding.rsindex 8350eff10..d6e6e8529 100644--- a/uzu/src/backends/metal/graph/embedding.rs+++ b/uzu/src/backends/metal/graph/embedding.rs@@ -1,119 +1,30 @@-#![allow(dead_code)]-use std::rc::Rc;--use mpsgraph::{Graph, GraphCallOps, GraphGatherOps, GraphMatrixOps, GraphTensorShapeOps, Tensor};-use objc2::rc::Retained;--use super::{- super::MTLContext,- GraphConstructionError,- common::{load_constant, shaped_type},+use mpsgraph::{+ Graph, GraphGatherOps, GraphMatrixOps, GraphQuantizationOps, Tensor,};-use crate::{config::EmbeddingConfig, parameters::ParameterTree};--pub enum EmbeddingParams {- Tied(Retained<Tensor>),- Untied {- input_weights: Retained<Tensor>,- output_weights: Retained<Tensor>,- },-}--pub fn embedding_params(- graph: &Graph,- config: &EmbeddingConfig,- vocabulary_size: usize,- model_dim: usize,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<EmbeddingParams, GraphConstructionError> {- match config {- EmbeddingConfig::Tied {- common: _,- precision,- } => {- let weights = load_constant(- graph,- ¶meter_tree.subtree("embedding").unwrap(),- "token_embeddings",- &[vocabulary_size, model_dim],- (*precision).into(),- )?;- Ok(EmbeddingParams::Tied(weights))- },- _ => {- unimplemented!()- },- }-}+use objc2::rc::Retained;-pub fn embed_callable_subgraph(- graph: &Graph,- config: &EmbeddingConfig,- embeddings_callable_name: &str,- embedding_dims: [isize; 2],- input: &Tensor,-) -> Result<Retained<Tensor>, GraphConstructionError> {- match config {- EmbeddingConfig::Tied {- common: _,- precision,- } => {- let embeddings_shaped_type =- shaped_type(&embedding_dims, (*precision).into());- if let Some(embeddings) = graph- .call(- embeddings_callable_name,- &[],- &[&embeddings_shaped_type],- None,- )- .first()- {- let result = graph.gather(embeddings, input, 0, 0, None);- Ok(result)- } else {- return Err(GraphConstructionError::CallableResultError {- name: embeddings_callable_name.to_string(),- });- }- },- _ => {- unimplemented!()- },- }-}+use super::GraphConstructionError;+use crate::config::EmbeddingConfig;-pub fn embed_subgraph(- graph: &Graph,- config: &EmbeddingConfig,- vocabulary_size: usize,- model_dim: usize,- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,+pub fn embeddings_dequantize_weights_subgraph(+ graph: &Retained<Graph>,+ weights: &Retained<Tensor>,+ scales: &Retained<Tensor>,) -> Result<Retained<Tensor>, GraphConstructionError> {- match config {- EmbeddingConfig::Tied {- common: _,- precision,- } => {- let embeddings = load_constant(- graph,- ¶meter_tree.subtree("embedding").unwrap(),- "token_embeddings",- &[vocabulary_size, model_dim],- (*precision).into(),- )- .unwrap();- let result = graph.gather(&embeddings, input, 0, 0, None);- Ok(result)- },- _ => {- unimplemented!()- },- }-}--pub fn embed_placeholder_weights_subgraph(+ let result = graph+ .dequantize_with_scale_tensor_zero_point_and_axis(+ &weights,+ &scales,+ 0.0,+ scales.data_type(),+ 0,+ None,+ )+ .unwrap();+ Ok(result)+}++pub fn embeddings_embed_subgraph(graph: &Retained<Graph>,config: &EmbeddingConfig,input: &Retained<Tensor>,@@ -130,95 +41,11 @@ pub fn embed_placeholder_weights_subgraph(}}-pub fn readout_callable_subgraph(- graph: &Graph,- config: &EmbeddingConfig,- embeddings_callable_name: &str,- embedding_dims: [isize; 2],- input: &Tensor,-) -> Result<Retained<Tensor>, GraphConstructionError> {- match config {- EmbeddingConfig::Tied {- common: _,- precision,- } => {- let embeddings_shaped_type =- shaped_type(&embedding_dims, (*precision).into());- if let Some(embeddings) = graph- .call(- embeddings_callable_name,- &[],- &[&embeddings_shaped_type],- None,- )- .first()- {- let embeddings_transposed =- graph.transpose(embeddings, &[1, 0], None);- let result = graph.matmul(- input,- &embeddings_transposed,- None,- );- Ok(result)- } else {- return Err(GraphConstructionError::CallableResultError {- name: embeddings_callable_name.to_string(),- });- }- },- _ => {- unimplemented!()- },- }-}--pub fn readout_subgraph(- graph: &Graph,- config: &EmbeddingConfig,- vocabulary_size: usize,- model_dim: usize,- input: &Tensor,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- match config {- EmbeddingConfig::Tied {- common: _,- precision,- } => {- let embeddings = load_constant(- graph,- ¶meter_tree.subtree("embedding").unwrap(),- "token_embeddings",- &[vocabulary_size, model_dim],- (*precision).into(),- )- .unwrap();- let embeddings_transposed =- graph.transpose(&embeddings, &[1, 0], None);- let result =- graph.matmul(input, &embeddings_transposed, None);- Ok(result)- },- _ => {- unimplemented!()- },- }-}--pub fn readout_placeholder_weights_subgraph(+pub fn embeddings_readout_subgraph(graph: &Retained<Graph>,input: &Retained<Tensor>,weights: &Retained<Tensor>,- transpose_weights: bool,) -> Result<Retained<Tensor>, GraphConstructionError> {- if transpose_weights {- let embeddings_transposed = graph.transpose(weights, &[1, 0], None);- let result =- graph.matmul(input, &embeddings_transposed, None);- Ok(result)- } else {- let result = graph.matmul(input, weights, None);- Ok(result)- }+ let result = graph.matmul(input, weights, None);+ Ok(result)}diff --git a/uzu/src/backends/metal/graph/linear.rs b/uzu/src/backends/metal/graph/linear.rsindex 23630c811..49af38f39 100644--- a/uzu/src/backends/metal/graph/linear.rs+++ b/uzu/src/backends/metal/graph/linear.rs@@ -8,129 +8,325 @@ use objc2::rc::Retained;use super::{super::MTLContext, GraphConstructionError, load_constant};use crate::{DataType,- config::{ConfigDataType, LinearConfig},+ config::{+ ConfigDataType, LinearConfig, QuantizationConfig, QuantizationMode,+ },parameters::ParameterTree,};-pub fn linear_subgraph<const N: usize>(+fn full_precision_weights_subgraph<const N: usize>(graph: &Graph,- config: &LinearConfig,+ precision: DataType,input_dim: usize,output_dims: [usize; N],- has_biases: bool,- input: &Tensor,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Result<Retained<Tensor>, GraphConstructionError> {- match config {- LinearConfig::FullPrecision {- precision,- } => {- let output_dim_sum: usize = output_dims.iter().sum();- let weights = load_constant(+ let output_dim_sum: usize = output_dims.iter().sum();+ load_constant(+ graph,+ parameter_tree,+ "weights",+ &[input_dim, output_dim_sum],+ precision,+ )+}++fn quantized_weights_subgraph<const N: usize>(+ graph: &Graph,+ config: &QuantizationConfig,+ input_dim: usize,+ output_dims: [usize; N],+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Retained<Tensor>, GraphConstructionError> {+ if config.weight_quantization_mode != QuantizationMode::UInt4 {+ unimplemented!("8-bit quantization is not yet implemented.");+ }+ let output_dim_sum: usize = output_dims.iter().sum();++ let weights = load_constant(+ graph,+ parameter_tree,+ "weights",+ &[input_dim, output_dim_sum],+ DataType::U4,+ )?;++ let scales = load_constant(+ graph,+ parameter_tree,+ "scales",+ &[input_dim, output_dim_sum / config.group_size],+ config.activation_precision.into(),+ )?;++ let zero_points = load_constant(+ graph,+ parameter_tree,+ "zero_points",+ &[input_dim, output_dim_sum / config.group_size],+ DataType::U4,+ )?;++ Ok(graph+ .dequantize_with_scale_tensor_and_zero_point_tensor(+ &weights,+ &scales,+ &zero_points,+ <ConfigDataType as Into<DataType>>::into(+ config.activation_precision,+ )+ .into(),+ None,+ )+ .unwrap())+}++fn biases_subgraph<const N: usize>(+ graph: &Graph,+ precision: DataType,+ output_dims: [usize; N],+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Retained<Tensor>, GraphConstructionError> {+ let output_dim_sum: usize = output_dims.iter().sum();+ load_constant(graph, parameter_tree, "biases", &[output_dim_sum], precision)+}++struct LoraWeights {+ down_weights: Retained<Tensor>,+ up_weights: Box<[Retained<Tensor>]>,+}++fn lora_weights_subgraph<const N: usize>(+ graph: &Graph,+ precision: DataType,+ input_dim: usize,+ output_dims: [usize; N],+ lora_rank: usize,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<LoraWeights, GraphConstructionError> {+ let down_weights = load_constant(+ graph,+ parameter_tree,+ "down_weights",+ &[input_dim, lora_rank * N],+ precision,+ )?;+ let up_weights = output_dims+ .iter()+ .enumerate()+ .map(|(i, output_dim)| {+ load_constant(graph,parameter_tree,- "weights",- &[input_dim, output_dim_sum],- (*precision).into(),- )?;+ &format!("up_weights.{}", i),+ &[lora_rank, *output_dim],+ precision,+ )+ })+ .collect::<Result<Box<[_]>, _>>()?;+ Ok(LoraWeights {+ down_weights,+ up_weights: up_weights,+ })+}- let matmul = graph.transpose(+fn full_precision_matmul_subgraph<const N: usize>(+ graph: &Graph,+ precision: DataType,+ input_dim: usize,+ output_dims: [usize; N],+ input: &Tensor,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Retained<Tensor>, GraphConstructionError> {+ let weights = full_precision_weights_subgraph::<N>(+ graph,+ precision,+ input_dim,+ output_dims,+ parameter_tree,+ )?;+ Ok(graph.transpose(+ &graph.matmul(+ &graph.transpose(&weights, &[1, 0], None),+ &graph.transpose(input, &[1, 0], None),+ None,+ ),+ &[1, 0],+ None,+ ))+}++fn quantized_matmul_subgraph<const N: usize>(+ graph: &Graph,+ config: &QuantizationConfig,+ input_dim: usize,+ output_dims: [usize; N],+ input: &Tensor,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Retained<Tensor>, GraphConstructionError> {+ let weights = quantized_weights_subgraph::<N>(+ graph,+ config,+ input_dim,+ output_dims,+ parameter_tree,+ )?;+ Ok(graph.matmul(input, &weights, None))+}++fn lora_subgraph<const N: usize>(+ graph: &Graph,+ precision: DataType,+ lora_rank: usize,+ input_dim: usize,+ output_dims: [usize; N],+ input: &Tensor,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Retained<Tensor>, GraphConstructionError> {+ let lora_weights = lora_weights_subgraph(+ graph,+ precision,+ input_dim,+ output_dims,+ lora_rank,+ parameter_tree,+ )?;++ let lora_inner = graph.transpose(+ &graph.matmul(+ &graph.transpose(&lora_weights.down_weights, &[1, 0], None),+ &graph.transpose(input, &[1, 0], None),+ None,+ ),+ &[1, 0],+ None,+ );++ let lora_inner_splits =+ graph.split(&lora_inner, output_dims.len(), 1, None);+ assert_eq!(lora_inner_splits.len(), lora_weights.up_weights.len());++ let lora_outputs = lora_inner_splits+ .iter()+ .zip(lora_weights.up_weights.iter())+ .map(|(lora_inner_split, up_weights)| {+ graph.transpose(&graph.matmul(- &graph.transpose(&weights, &[1, 0], None),- &graph.transpose(input, &[1, 0], None),+ &graph.transpose(&up_weights, &[1, 0], None),+ &graph.transpose(&lora_inner_split, &[1, 0], None),None,),&[1, 0],None,- );-- if has_biases {- let biases = load_constant(- graph,- parameter_tree,- "biases",- &[output_dim_sum],- (*precision).into(),- )?;-- let result = graph.add(&matmul, &biases, None);-- Ok(result)- } else {- Ok(matmul)- }- },- LinearConfig::Quantized(quantization_config) => {- let output_dim_sum: usize = output_dims.iter().sum();- let group_size = quantization_config.group_size;- let activation_precision = quantization_config.activation_precision;+ )+ })+ .collect::<Box<[Retained<Tensor>]>>();- let weights = load_constant(- graph,- parameter_tree,- "weights",- &[output_dim_sum, input_dim],- DataType::U4,- )?;+ Ok(graph.concat_tensors(+ &lora_outputs+ .iter()+ .map(|tensor| &**tensor)+ .collect::<Box<[&Tensor]>>(),+ 1,+ None,+ ))+}- let scales = load_constant(- graph,- parameter_tree,- "scales",- &[output_dim_sum, input_dim / group_size],- activation_precision.into(),- )?;+fn qlora_matmul_subgraph<const N: usize>(+ graph: &Graph,+ lora_rank: usize,+ lora_scale: f32,+ config: &QuantizationConfig,+ input_dim: usize,+ output_dims: [usize; N],+ input: &Tensor,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Retained<Tensor>, GraphConstructionError> {+ let quantized_matmul_result = quantized_matmul_subgraph::<N>(+ graph,+ config,+ input_dim,+ output_dims,+ input,+ parameter_tree,+ )?;+ let lora_result = lora_subgraph::<N>(+ graph,+ config.activation_precision.into(),+ lora_rank,+ input_dim,+ output_dims,+ input,+ parameter_tree,+ )?;+ let scaled_lora_result = graph.multiply(+ &lora_result,+ &graph.constant_with_scalar(+ lora_scale as f64,+ <ConfigDataType as Into<DataType>>::into(+ config.activation_precision,+ )+ .into(),+ ),+ None,+ );+ Ok(graph.add(&quantized_matmul_result, &scaled_lora_result, None))+}- let zero_points = load_constant(+pub fn linear_subgraph<const N: usize>(+ graph: &Graph,+ config: &LinearConfig,+ input_dim: usize,+ output_dims: [usize; N],+ has_biases: bool,+ input: &Tensor,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Retained<Tensor>, GraphConstructionError> {+ let matmul_result = match config {+ LinearConfig::FullPrecision {+ precision,+ } => full_precision_matmul_subgraph(+ graph,+ (*precision).into(),+ input_dim,+ output_dims,+ input,+ parameter_tree,+ )?,+ LinearConfig::Quantized(quantization_config) => {+ quantized_matmul_subgraph(graph,+ quantization_config,+ input_dim,+ output_dims,+ input,parameter_tree,- "zero_points",- &[output_dim_sum, input_dim / group_size],- DataType::U4,- )?;-- let dequantized_weights = graph- .dequantize_with_scale_tensor_and_zero_point_tensor(- &weights,- &scales,- &zero_points,- <ConfigDataType as Into<DataType>>::into(- activation_precision,- )- .into(),- None,- )- .unwrap();-- let matmul = graph.matmul(- &input,- &graph.transpose(&dequantized_weights, &[1, 0], None),- None,- );-- if has_biases {- let biases = load_constant(- graph,- parameter_tree,- "biases",- &[output_dim_sum],- activation_precision.into(),- )?;-- let result = graph.add(&matmul, &biases, None);-- Ok(result)- } else {- Ok(matmul)- }+ )?},LinearConfig::QLoRA {- ..- } => {- // QLoRA linear layer implementation- unimplemented!(- "QLoRA linear layer implementation not yet available"- )- },+ quantization: quantization_config,+ lora_rank,+ lora_scale,+ } => qlora_matmul_subgraph(+ graph,+ *lora_rank,+ *lora_scale,+ quantization_config,+ input_dim,+ output_dims,+ input,+ parameter_tree,+ )?,+ };+ if has_biases {+ let biases = biases_subgraph(+ graph,+ config.activation_precision().into(),+ output_dims,+ parameter_tree,+ )?;+ Ok(graph.add(&matmul_result, &biases, None))+ } else {+ Ok(matmul_result)}}diff --git a/uzu/src/backends/metal/graph/mod.rs b/uzu/src/backends/metal/graph/mod.rsindex fae91049b..27a626eb1 100644--- a/uzu/src/backends/metal/graph/mod.rs+++ b/uzu/src/backends/metal/graph/mod.rs@@ -11,9 +11,7 @@ mod rms_norm;pub use rms_norm::rms_norm_subgraph;mod embedding;pub use embedding::{- EmbeddingParams, embed_callable_subgraph,- embed_placeholder_weights_subgraph, embed_subgraph, embedding_params,- readout_callable_subgraph, readout_placeholder_weights_subgraph,- readout_subgraph,+ embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,+ embeddings_readout_subgraph,};mod rope;diff --git a/uzu/src/config/common.rs b/uzu/src/config/common.rsindex aa7d54783..f2f1896ce 100644--- a/uzu/src/config/common.rs+++ b/uzu/src/config/common.rs@@ -46,6 +46,16 @@ pub enum QuantizationMode {UInt8,}+impl Into<DataType> for QuantizationMode {+ fn into(self) -> DataType {+ match self {+ QuantizationMode::UInt4 => DataType::U4,+ QuantizationMode::Int8 => DataType::I8,+ QuantizationMode::UInt8 => DataType::U8,+ }+ }+}+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Copy, Clone)]#[serde(rename_all = "lowercase")]pub enum Activation {diff --git a/uzu/tests/parameters_loader_test.rs b/uzu/tests/parameters_loader_test.rsindex a56647d97..36a10606b 100644--- a/uzu/tests/parameters_loader_test.rs+++ b/uzu/tests/parameters_loader_test.rs@@ -14,12 +14,12 @@ fn test_parameter_loader_basic() {let loader = ParameterLoader::new(&file, &context).expect("create loader");let embeddings =- loader.get("embedding.token_embeddings").expect("load embeddings");+ loader.get("embedding.weights").expect("weights embeddings");let view = embeddings.as_view::<f16>().unwrap();assert!(is_close!(view[[5usize, 3usize]], f16::from_f32(-0.01819)));// tree API checklet subtree = loader.tree().subtree("embedding").unwrap();- let same = subtree.leaf("token_embeddings").unwrap();+ let same = subtree.leaf("weights").unwrap();assert_eq!(view, same.as_view::<f16>().unwrap());}