A high-performance inference engine for AI models
generalize full precision embedding lookup
eugeneb5044b356652
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rsdeleted file mode 100644index 0b486cfba..000000000--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs+++ /dev/null@@ -1,121 +0,0 @@-use super::{- super::{EncodableBlock, Metal},- EmbeddingError,-};-use crate::{- DataType,- backends::{- common::kernel::FullPrecisionEmbeddingLookupKernel,- metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError,- ProtocolObject, Retained, kernel::dsl::FullPrecisionEmbeddingLookupMetalKernel,- },- },- encodable_block::EncodingParameters,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,-};--pub struct FullPrecisionEmbeddingLookup {- kernel: FullPrecisionEmbeddingLookupMetalKernel,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- vocab_size: u32,- model_dim: u32,- input_scale: f32,-}--impl FullPrecisionEmbeddingLookup {- pub fn new(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- input_scale: Option<f32>,- parameter_tree: &ParameterTree<MTLContext>,- ) -> Result<Self, EmbeddingError> {- let kernel = FullPrecisionEmbeddingLookupMetalKernel::new(mtl_context, data_type.into())?;-- let weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))- })?,- };-- if weights.shape() != [vocab_size, model_dim] {- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(- "Embedding lookup weights shape mismatch: got {:?}, \- expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim- ))));- }-- if weights.data_type() != data_type {- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(- "Weights dtype mismatch: got {:?}, expected {:?}",- weights.data_type(),- data_type- ))));- }-- let weights_buffer = weights.buffer().to_owned();-- Ok(Self {- kernel,- weights_buffer: weights_buffer.into(),- vocab_size: vocab_size as u32,- model_dim: model_dim as u32,- input_scale: input_scale.unwrap_or(1.0),- })- }-}--impl EncodableBlock<Metal> for FullPrecisionEmbeddingLookup {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &encoder);- encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<Metal>,- _parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) {- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);- let batch_size = state.active_suffix_length();- let token_ids_array_mut = arrays[0].borrow_mut();- let output_array_mut = arrays[1].borrow_mut();-- let token_ids_buffer = token_ids_array_mut.buffer();- let output_buffer = output_array_mut.buffer();-- self.kernel.encode(- token_ids_buffer,- &self.weights_buffer,- output_buffer,- batch_size as u32,- self.vocab_size,- self.model_dim,- self.input_scale,- encoder,- )- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rsindex 895b26ca0..c9af38b13 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs@@ -1,13 +1,16 @@//! Embedding encodables (quantized and full-precision).mod error;-mod full_precision_lookup;mod full_precision_readout;mod quantized_lookup;mod quantized_readout;pub use error::EmbeddingError;-pub use full_precision_lookup::FullPrecisionEmbeddingLookup;pub use full_precision_readout::FullPrecisionEmbeddingReadout;pub use quantized_lookup::QuantizedEmbeddingLookup;pub use quantized_readout::QuantizedEmbeddingReadout;++use super::super::Metal;+use crate::encodable_block::FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup;++pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rsnew file mode 100644index 000000000..bab9f78a4--- /dev/null+++ b/crates/uzu/src/encodable_block/full_precision_embedding_lookup.rs@@ -0,0 +1,114 @@+//! Full precision embedding lookup encodable.++use thiserror::Error;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ DataType,+ backends::common::{+ Backend,+ kernel::{FullPrecisionEmbeddingLookupKernel, Kernels},+ },+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum FullPrecisionEmbeddingLookupError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loading error: {0}")]+ ParameterError(ParameterLoaderError),+ #[error(+ "Embedding lookup weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"+ )]+ InvalidWeightsShape {+ got: Box<[usize]>,+ expected_vocab_size: usize,+ expected_model_dim: usize,+ },+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidWeightsDataType {+ got: DataType,+ expected: DataType,+ },+}++pub struct FullPrecisionEmbeddingLookup<B: Backend> {+ kernel: <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel,+ weights_buffer: B::NativeBuffer,+ vocab_size: u32,+ model_dim: u32,+ input_scale: f32,+}++impl<B: Backend> FullPrecisionEmbeddingLookup<B> {+ pub fn new(+ context: &B::Context,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ input_scale: Option<f32>,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, FullPrecisionEmbeddingLookupError<B>> {+ let kernel = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)+ .map_err(FullPrecisionEmbeddingLookupError::BackendError)?;++ let weights = parameter_tree+ .leaf("weights")+ .or_else(|_| parameter_tree.leaf("input_weights"))+ .map_err(FullPrecisionEmbeddingLookupError::ParameterError)?;++ if weights.shape() != [vocab_size, model_dim] {+ return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsShape {+ got: weights.shape().to_vec().into_boxed_slice(),+ expected_vocab_size: vocab_size,+ expected_model_dim: model_dim,+ });+ }++ if weights.data_type() != data_type {+ return Err(FullPrecisionEmbeddingLookupError::InvalidWeightsDataType {+ got: weights.data_type(),+ expected: data_type,+ });+ }++ Ok(Self {+ kernel,+ weights_buffer: weights.buffer().clone(),+ vocab_size: vocab_size as u32,+ model_dim: model_dim as u32,+ input_scale: input_scale.unwrap_or(1.0),+ })+ }+}++impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingLookup<B> {+ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState<B>,+ _parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,+ ) {+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);+ let batch_size = state.active_suffix_length();+ let token_ids_array = arrays[0].borrow_mut();+ let output_array = arrays[1].borrow_mut();++ self.kernel.encode(+ token_ids_array.buffer(),+ &self.weights_buffer,+ output_array.buffer(),+ batch_size as u32,+ self.vocab_size,+ self.model_dim,+ self.input_scale,+ encoder,+ );+ }+}diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 478b40a4f..0a0f6a2e4 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -4,6 +4,7 @@ use crate::{};mod activation;+mod full_precision_embedding_lookup;mod layer_norm;mod normalization;mod pooling;@@ -15,6 +16,7 @@ mod tensor_add_swap;mod tensor_copy;pub use activation::Activation;+pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};pub use layer_norm::{LayerNorm, LayerNormError};pub use normalization::{Normalization, NormalizationError};pub use pooling::Pooling;