Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generalize full precision embedding lookup

eugeneb5044b356652
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
deleted file mode 100644
index 0b486cfba..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-use super::{
- super::{EncodableBlock, Metal},
- EmbeddingError,
-};
-use crate::{
- DataType,
- backends::{
- common::kernel::FullPrecisionEmbeddingLookupKernel,
- metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,
- ProtocolObject, Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,
- },
- },
- encodable_block::EncodingParameters,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
-};
-
-pub struct FullPrecisionEmbeddingLookup {
- kernel: FullPrecisionEmbeddingLookupMetalKernel,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- vocab_size: u32,
- model_dim: u32,
- input_scale: f32,
-}
-
-impl FullPrecisionEmbeddingLookup {
- pub fn new(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- input_scale: Option<f32>,
- parameter_tree: &ParameterTree<MTLContext>,
- ) -> Result<Self, EmbeddingError> {
- let kernel = FullPrecisionEmbeddingLookupMetalKernel::new(mtl_context, data_type.into())?;
-
- let weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))
- })?,
- };
-
- if weights.shape() != [vocab_size, model_dim] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
- "Embedding lookup weights shape mismatch: got {:?}, \
- expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim
- ))));
- }
-
- if weights.data_type() != data_type {
- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
- "Weights dtype mismatch: got {:?}, expected {:?}",
- weights.data_type(),
- data_type
- ))));
- }
-
- let weights_buffer = weights.buffer().to_owned();
-
- Ok(Self {
- kernel,
- weights_buffer: weights_buffer.into(),
- vocab_size: vocab_size as u32,
- model_dim: model_dim as u32,
- input_scale: input_scale.unwrap_or(1.0),
- })
- }
-}
-
-impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &encoder);
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<Metal>,
- _parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) {
- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
- let batch_size = state.active_suffix_length();
- let token_ids_array_mut = arrays[0].borrow_mut();
- let output_array_mut = arrays[1].borrow_mut();
-
- let token_ids_buffer = token_ids_array_mut.buffer();
- let output_buffer = output_array_mut.buffer();
-
- self.kernel.encode(
- token_ids_buffer,
- &self.weights_buffer,
- output_buffer,
- batch_size as u32,
- self.vocab_size,
- self.model_dim,
- self.input_scale,
- encoder,
- )
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
index 895b26ca0..c9af38b13 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
@@ -1,13 +1,16 @@
//! Embedding encodables (quantized and full-precision).
mod error;
-mod full_precision_lookup;
mod full_precision_readout;
mod quantized_lookup;
mod quantized_readout;
pub use error::EmbeddingError;
-pub use full_precision_lookup::FullPrecisionEmbeddingLookup;
pub use full_precision_readout::FullPrecisionEmbeddingReadout;
pub use quantized_lookup::QuantizedEmbeddingLookup;
pub use quantized_readout::QuantizedEmbeddingReadout;
+
+use super::super::Metal;
+use crate::encodable_block::FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup;
+
+pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
new file mode 100644
index 000000000..bab9f78a4
--- /dev/null
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs
@@ -0,0 +1,114 @@
+//! Full precision embedding lookup encodable.
+
+use thiserror::Error;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend,
+ kernel::{FullPrecisionEmbeddingLookupKernel, Kernels},
+ },
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum FullPrecisionEmbeddingLookupError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(ParameterLoaderError),
+ #[error(
+ "Embedding lookup weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"
+ )]
+ InvalidWeightsShape {
+ got: Box<[usize]>,
+ expected_vocab_size: usize,
+ expected_model_dim: usize,
+ },
+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidWeightsDataType {
+ got: DataType,
+ expected: DataType,
+ },
+}
+
+pub struct FullPrecisionEmbeddingLookup<B: Backend> {
+ kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,
+ weights_buffer: B::NativeBuffer,
+ vocab_size: u32,
+ model_dim: u32,
+ input_scale: f32,
+}
+
+impl<B: Backend> FullPrecisionEmbeddingLookup<B> {
+ pub fn new(
+ context: &B::Context,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ input_scale: Option<f32>,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, FullPrecisionEmbeddingLookupError<B>> {
+ let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
+ .map_err(FullPrecisionEmbeddingLookupError::BackendError)?;
+
+ let weights = parameter_tree
+ .leaf("weights")
+ .or_else(|_| parameter_tree.leaf("input_weights"))
+ .map_err(FullPrecisionEmbeddingLookupError::ParameterError)?;
+
+ if weights.shape() != [vocab_size, model_dim] {
+ return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsShape {
+ got: weights.shape().to_vec().into_boxed_slice(),
+ expected_vocab_size: vocab_size,
+ expected_model_dim: model_dim,
+ });
+ }
+
+ if weights.data_type() != data_type {
+ return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsDataType {
+ got: weights.data_type(),
+ expected: data_type,
+ });
+ }
+
+ Ok(Self {
+ kernel,
+ weights_buffer: weights.buffer().clone(),
+ vocab_size: vocab_size as u32,
+ model_dim: model_dim as u32,
+ input_scale: input_scale.unwrap_or(1.0),
+ })
+ }
+}
+
+impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState<B>,
+ _parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
+ ) {
+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
+ let batch_size = state.active_suffix_length();
+ let token_ids_array = arrays[0].borrow_mut();
+ let output_array = arrays[1].borrow_mut();
+
+ self.kernel.encode(
+ token_ids_array.buffer(),
+ &self.weights_buffer,
+ output_array.buffer(),
+ batch_size as u32,
+ self.vocab_size,
+ self.model_dim,
+ self.input_scale,
+ encoder,
+ );
+ }
+}
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 478b40a4f..0a0f6a2e4 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -4,6 +4,7 @@ use crate::{
};
mod activation;
+mod full_precision_embedding_lookup;
mod layer_norm;
mod normalization;
mod pooling;
@@ -15,6 +16,7 @@ mod tensor_add_swap;
mod tensor_copy;
pub use activation::Activation;
+pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};
pub use layer_norm::{LayerNorm, LayerNormError};
pub use normalization::{Normalization, NormalizationError};
pub use pooling::Pooling;