Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

decouple linear into a mod

eugened13297a9ccbe
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
new file mode 100644
index 000000000..155324388
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -0,0 +1,231 @@
+use std::{cell::RefCell, rc::Rc};
+
+use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ encodable_block::{EncodableBlock, EncodingParameters},
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::{
+ KernelDataType, TensorAddBias,
+ matmul::{MatmulArguments, MatmulKernel},
+ },
+ },
+ device::array::Array,
+ parameters::ParameterTree,
+};
+
+pub struct FullPrecisionLinear {
+ kernel: RefCell<MatmulKernel>,
+ bias_add_kernel: Option<TensorAddBias>,
+ biases_buffer: Option<MTLBuffer>,
+ weights_buffer: MTLBuffer,
+ input_dim: usize,
+ output_dim: usize,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+}
+
+impl FullPrecisionLinear {
+ pub fn new(
+ mtl_context: &MTLContext,
+ precision: DataType,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, MTLError> {
+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for full precision linear kernel: {:?}",
+ precision
+ )));
+ }
+
+ let mut weights = parameter_tree.leaf("weights").map_err(|e| {
+ MTLError::Generic(format!("Failed to load weights: {:?}", e))
+ })?;
+
+ let w_shape = weights.shape();
+ if w_shape != [output_dim, input_dim] {
+ return Err(MTLError::Generic(format!(
+ "Unexpected weights shape: got {:?}, expected [{}, {}]",
+ w_shape, output_dim, input_dim
+ )));
+ }
+
+ if weights.data_type() != precision {
+ return Err(MTLError::Generic(format!(
+ "Weights dtype mismatch: got {:?}, expected {:?}",
+ weights.data_type(),
+ precision
+ )));
+ }
+
+ let weights_buffer: MTLBuffer =
+ unsafe { weights.mtl_buffer() }.to_owned();
+
+ let (bias_add_kernel, biases_buffer) =
+ match parameter_tree.leaf("biases") {
+ Ok(mut biases) => {
+ if biases.shape() != [output_dim] {
+ return Err(MTLError::Generic(format!(
+ "Bias shape mismatch: got {:?}, expected [{:?}]",
+ biases.shape(),
+ output_dim
+ )));
+ }
+ if biases.data_type() != precision {
+ return Err(MTLError::Generic(format!(
+ "Bias dtype mismatch: got {:?}, expected {:?}",
+ biases.data_type(),
+ precision
+ )));
+ }
+ let bias_add_kernel = Some(TensorAddBias::new(
+ mtl_context,
+ KernelDataType::from(precision),
+ )?);
+ let biases_buffer: MTLBuffer =
+ unsafe { biases.mtl_buffer() }.to_owned();
+ (bias_add_kernel, Some(biases_buffer))
+ },
+ Err(_) => (None, None),
+ };
+
+ let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;
+
+ Ok(Self {
+ kernel: RefCell::new(kernel),
+ bias_add_kernel,
+ biases_buffer,
+ weights_buffer,
+ input_dim,
+ output_dim,
+ input_array_id,
+ output_array_id,
+ })
+ }
+}
+
+impl EncodableBlock for FullPrecisionLinear {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let batch_size = state.active_suffix_length();
+
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = MatmulArguments {
+ a: input_buffer,
+ b: &self.weights_buffer,
+ d: output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.input_dim as i32,
+ output_dim: self.output_dim as i32,
+ lda: self.input_dim as i32,
+ ldb: self.input_dim as i32,
+ ldd: self.output_dim as i32,
+ batch_count: 1,
+ };
+
+ self.kernel
+ .borrow_mut()
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
+
+ encoder.end_encoding();
+
+ if let (Some(bias_add), Some(bias_buf)) =
+ (&self.bias_add_kernel, &self.biases_buffer)
+ {
+ let total_len = batch_size * self.output_dim;
+ let retained_cb = root_command_buffer.to_owned();
+ bias_add.encode_into_command_buffer(
+ &output_buffer,
+ bias_buf,
+ &output_buffer,
+ self.output_dim,
+ total_len,
+ &retained_cb,
+ parameters.predicate,
+ );
+ }
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let batch_size = state.active_suffix_length();
+
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let args = MatmulArguments {
+ a: input_buffer,
+ b: &self.weights_buffer,
+ d: output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.input_dim as i32,
+ output_dim: self.output_dim as i32,
+ lda: self.input_dim as i32,
+ ldb: self.input_dim as i32,
+ ldd: self.output_dim as i32,
+ batch_count: 1,
+ };
+
+ self.kernel
+ .borrow_mut()
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
+
+ if let (Some(bias_add), Some(bias_buf)) =
+ (&self.bias_add_kernel, &self.biases_buffer)
+ {
+ let total_len = batch_size * self.output_dim;
+ bias_add.encode_with_encoder(
+ &output_buffer,
+ bias_buf,
+ &output_buffer,
+ self.output_dim,
+ total_len,
+ encoder,
+ parameters.predicate,
+ );
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs
new file mode 100644
index 000000000..139ecef36
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs
@@ -0,0 +1,5 @@
+mod full_precision;
+mod quantized;
+
+pub use full_precision::FullPrecisionLinear;
+pub use quantized::QuantizedLinear;
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
similarity index 62%
rename from crates/uzu/src/backends/metal/encodable_block/linear.rs
rename to crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
index 17293ca29..9fab0314e 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
@@ -1,19 +1,16 @@
-//! Linear encodables implemented with Metal kernels.
-
-use std::{cell::RefCell, rc::Rc};
+use std::rc::Rc;
use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::{EncodableBlock, EncodingParameters};
use crate::{
- Array, DataType,
+ DataType,
backends::metal::{
MTLContext, MTLError,
+ encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::{ArrayId, ForwardPassState},
kernel::{
KernelDataType, TensorAddBias,
- matmul::{MatmulArguments, MatmulKernel},
quant_matmul::{
QuantizationType, QuantizedMatmulArguments,
QuantizedMatmulKernel,
@@ -21,221 +18,10 @@ use crate::{
},
},
config::QuantizationConfig,
+ device::array::Array,
parameters::ParameterTree,
};
-pub struct FullPrecisionLinear {
- kernel: RefCell<MatmulKernel>,
- bias_add_kernel: Option<TensorAddBias>,
- biases_buffer: Option<MTLBuffer>,
- weights_buffer: MTLBuffer,
- input_dim: usize,
- output_dim: usize,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
-}
-
-impl FullPrecisionLinear {
- pub fn new(
- mtl_context: &MTLContext,
- precision: DataType,
- input_dim: usize,
- output_dim: usize,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- ) -> Result<Self, MTLError> {
- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for full precision linear kernel: {:?}",
- precision
- )));
- }
-
- let mut weights = parameter_tree.leaf("weights").map_err(|e| {
- MTLError::Generic(format!("Failed to load weights: {:?}", e))
- })?;
-
- let w_shape = weights.shape();
- if w_shape != [output_dim, input_dim] {
- return Err(MTLError::Generic(format!(
- "Unexpected weights shape: got {:?}, expected [{}, {}]",
- w_shape, output_dim, input_dim
- )));
- }
-
- if weights.data_type() != precision {
- return Err(MTLError::Generic(format!(
- "Weights dtype mismatch: got {:?}, expected {:?}",
- weights.data_type(),
- precision
- )));
- }
-
- let weights_buffer: MTLBuffer =
- unsafe { weights.mtl_buffer() }.to_owned();
-
- let (bias_add_kernel, biases_buffer) =
- match parameter_tree.leaf("biases") {
- Ok(mut biases) => {
- if biases.shape() != [output_dim] {
- return Err(MTLError::Generic(format!(
- "Bias shape mismatch: got {:?}, expected [{:?}]",
- biases.shape(),
- output_dim
- )));
- }
- if biases.data_type() != precision {
- return Err(MTLError::Generic(format!(
- "Bias dtype mismatch: got {:?}, expected {:?}",
- biases.data_type(),
- precision
- )));
- }
- let bias_add_kernel = Some(TensorAddBias::new(
- mtl_context,
- KernelDataType::from(precision),
- )?);
- let biases_buffer: MTLBuffer =
- unsafe { biases.mtl_buffer() }.to_owned();
- (bias_add_kernel, Some(biases_buffer))
- },
- Err(_) => (None, None),
- };
-
- let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;
-
- Ok(Self {
- kernel: RefCell::new(kernel),
- bias_add_kernel,
- biases_buffer,
- weights_buffer,
- input_dim,
- output_dim,
- input_array_id,
- output_array_id,
- })
- }
-}
-
-impl EncodableBlock for FullPrecisionLinear {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &MPSCommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
- let batch_size = state.active_suffix_length();
-
- let mut input_array_mut = arrays[0].borrow_mut();
- let mut output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = unsafe { input_array_mut.mtl_buffer() };
- let output_buffer = unsafe { output_array_mut.mtl_buffer() };
-
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
- let args = MatmulArguments {
- a: input_buffer,
- b: &self.weights_buffer,
- d: output_buffer,
- batch: batch_size as i32,
- input_dim: self.input_dim as i32,
- output_dim: self.output_dim as i32,
- lda: self.input_dim as i32,
- ldb: self.input_dim as i32,
- ldd: self.output_dim as i32,
- batch_count: 1,
- };
-
- self.kernel
- .borrow_mut()
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode matmul kernel");
-
- encoder.end_encoding();
-
- if let (Some(bias_add), Some(bias_buf)) =
- (&self.bias_add_kernel, &self.biases_buffer)
- {
- let total_len = batch_size * self.output_dim;
- let retained_cb = root_command_buffer.to_owned();
- bias_add.encode_into_command_buffer(
- &output_buffer,
- bias_buf,
- &output_buffer,
- self.output_dim,
- total_len,
- &retained_cb,
- parameters.predicate,
- );
- }
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState,
- encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
- let batch_size = state.active_suffix_length();
-
- let mut input_array_mut = arrays[0].borrow_mut();
- let mut output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = unsafe { input_array_mut.mtl_buffer() };
- let output_buffer = unsafe { output_array_mut.mtl_buffer() };
-
- let args = MatmulArguments {
- a: input_buffer,
- b: &self.weights_buffer,
- d: output_buffer,
- batch: batch_size as i32,
- input_dim: self.input_dim as i32,
- output_dim: self.output_dim as i32,
- lda: self.input_dim as i32,
- ldb: self.input_dim as i32,
- ldd: self.output_dim as i32,
- batch_count: 1,
- };
-
- self.kernel
- .borrow_mut()
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode matmul kernel");
-
- if let (Some(bias_add), Some(bias_buf)) =
- (&self.bias_add_kernel, &self.biases_buffer)
- {
- let total_len = batch_size * self.output_dim;
- bias_add.encode_with_encoder(
- &output_buffer,
- bias_buf,
- &output_buffer,
- self.output_dim,
- total_len,
- encoder,
- parameters.predicate,
- );
- }
- }
-}
-
pub struct QuantizedLinear {
kernel: QuantizedMatmulKernel,
bias_add_kernel: Option<TensorAddBias>,
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 2acd7b814..43ff39948 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -35,8 +35,8 @@ pub use attention::Attention;
pub use classifier_layer::ClassifierLayer;
pub use decoder::Decoder;
pub use embedding::{
- FullPrecisionEmbeddingReadout, QuantizedEmbeddingError,
- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,
+ FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout,
+ QuantizedEmbeddingError, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,
};
pub use encoding_parameters::EncodingParameters;
pub use layer::LayerExecutables;
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 53ed69431..9ef459df6 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -7,9 +7,9 @@ use mpsgraph::{
use objc2::rc::{Retained, autoreleasepool};
use super::{
- EncodableBlock, FullPrecisionEmbeddingReadout, FullPrecisionLinear,
- MlpBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,
- QuantizedLinear,
+ EncodableBlock, FullPrecisionEmbeddingLookup,
+ FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock, MoeBlock,
+ QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
@@ -23,10 +23,7 @@ use crate::{
},
kernel::mlp::MlpGateActMulEncodable,
},
- config::{
- DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig,
- QuantizationConfig,
- },
+ config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
parameters::ParameterTree,
};
@@ -51,16 +48,17 @@ pub fn linear_block<const N: usize>(
match config {
LinearConfig::Quantized(quant_config)
| LinearConfig::MLXQuantized(quant_config) => {
- return quantized_linear_block_custom(
+ let block = QuantizedLinear::new(
+ context,
quant_config,
input_dim,
out_sum,
- context,
parameter_tree,
input_array_id,
output_array_id,
)
- .unwrap();
+ .expect("Failed to create quantized linear kernel block");
+ return Box::new(block);
},
LinearConfig::FullPrecision {
precision,
@@ -126,27 +124,6 @@ pub fn linear_block<const N: usize>(
})
}
-pub fn quantized_linear_block_custom(
- config: &QuantizationConfig,
- input_dim: usize,
- output_dim: usize,
- context: &MTLContext,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
-) -> Result<Box<dyn EncodableBlock>, crate::backends::metal::MTLError> {
- let block = QuantizedLinear::new(
- context,
- config,
- input_dim,
- output_dim,
- parameter_tree,
- input_array_id,
- output_array_id,
- )?;
- Ok(Box::new(block))
-}
-
pub fn mlp_block(
config: &MLPConfig,
model_dim: usize,
@@ -266,146 +243,65 @@ pub fn embed_block(
compilation_descriptor: &CompilationDescriptor,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Box<dyn EncodableBlock> {
- let graph = Graph::new();
-
- let input_shape = [-1 as isize];
- let input_data_type = DataType::U64;
- let input_placeholder = placeholder(&graph, &input_shape, input_data_type);
- let input_shaped_type = shaped_type(&input_shape, input_data_type);
-
match &config.embedding_config {
EmbeddingConfig::Tied {
+ common,
precision,
- ..
} => {
- let weights_shape =
- [config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = (*precision).into();
- let weights_shaped_type =
- shaped_type(&weights_shape, weights_data_type);
- let weights_placeholder =
- placeholder(&graph, &weights_shape, weights_data_type);
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
- let output = embeddings_embed_subgraph(
- &graph,
- &config.embedding_config,
- &input_placeholder,
- &weights_placeholder,
+ let block = FullPrecisionEmbeddingLookup::new(
+ context,
+ (*precision).into(),
+ config.vocab_size,
+ config.model_dim,
+ common.input_scale,
+ &embeddings_tree,
)
- .unwrap();
-
- let feeds = HashMap::from([
- (&*input_placeholder, &*input_shaped_type),
- (&*weights_placeholder, &*weights_shaped_type),
- ]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(&compilation_descriptor),
- );
-
- let block = MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]
- .into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- ),
- );
+ .expect("Failed to create full precision embedding lookup");
Box::new(block)
},
EmbeddingConfig::Untied {
+ common,
precision,
- ..
} => {
- let weights_shape =
- [config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = (*precision).into();
- let weights_shaped_type =
- shaped_type(&weights_shape, weights_data_type);
- let weights_placeholder =
- placeholder(&graph, &weights_shape, weights_data_type);
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
- let output = embeddings_embed_subgraph(
- &graph,
- &config.embedding_config,
- &input_placeholder,
- &weights_placeholder,
+ let block = FullPrecisionEmbeddingLookup::new(
+ context,
+ (*precision).into(),
+ config.vocab_size,
+ config.model_dim,
+ common.input_scale,
+ &embeddings_tree,
)
- .unwrap();
-
- let feeds = HashMap::from([
- (&*input_placeholder, &*input_shaped_type),
- (&*weights_placeholder, &*weights_shaped_type),
- ]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(&compilation_descriptor),
- );
-
- let block = MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]
- .into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- ),
- );
+ .expect("Failed to create full precision embedding lookup");
Box::new(block)
},
EmbeddingConfig::MLXSemiQuantizedUntied {
+ common,
activation_precision,
..
} => {
- let weights_shape =
- [config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = (*activation_precision).into();
- let weights_shaped_type =
- shaped_type(&weights_shape, weights_data_type);
- let weights_placeholder =
- placeholder(&graph, &weights_shape, weights_data_type);
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
- let output = embeddings_embed_subgraph(
- &graph,
- &config.embedding_config,
- &input_placeholder,
- &weights_placeholder,
+ let block = FullPrecisionEmbeddingLookup::new(
+ context,
+ (*activation_precision).into(),
+ config.vocab_size,
+ config.model_dim,
+ common.input_scale,
+ &embeddings_tree,
)
- .unwrap();
-
- let feeds = HashMap::from([
- (&*input_placeholder, &*input_shaped_type),
- (&*weights_placeholder, &*weights_shaped_type),
- ]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(&compilation_descriptor),
- );
-
- let block = MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::TokenIds, ArrayId::EmbeddingsInputWeights]
- .into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- ),
- );
+ .expect("Failed to create full precision embedding lookup");
Box::new(block)
},
@@ -413,6 +309,13 @@ pub fn embed_block(
embedding_quantization_mode: _,
..
} => {
+ let graph = Graph::new();
+
+ let input_shape = [-1 as isize];
+ let input_data_type = DataType::U64;
+ let input_placeholder =
+ placeholder(&graph, &input_shape, input_data_type);
+ let input_shaped_type = shaped_type(&input_shape, input_data_type);
let weights_shape =
[config.vocab_size as isize, (config.model_dim / 2) as isize];
let weights_data_type = DataType::U8;