A high-performance inference engine for AI models
full precision lookup
eugeneec6f03e1b4f2
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rsnew file mode 100644index 000000000..5bd6c6ae1--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs@@ -0,0 +1,128 @@+use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer;++use super::QuantizedEmbeddingError;+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::embedding::{+ FullPrecisionEmbeddingLookupArguments,+ FullPrecisionEmbeddingLookupKernel,+ },+ },+ parameters::ParameterTree,+};++use super::super::{EncodableBlock, EncodingParameters};++pub struct FullPrecisionEmbeddingLookup {+ kernel: FullPrecisionEmbeddingLookupKernel,+ weights_buffer: MTLBuffer,+ vocab_size: u32,+ model_dim: u32,+ input_scale: f32,+}++impl FullPrecisionEmbeddingLookup {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ input_scale: Option<f32>,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ let kernel =+ FullPrecisionEmbeddingLookupKernel::new(mtl_context, data_type)?;++ let mut weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?,+ };++ if weights.shape() != [vocab_size, model_dim] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding lookup weights shape mismatch: got {:?}, \+ expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim+ )),+ ));+ }++ if weights.data_type() != data_type {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Weights dtype mismatch: got {:?}, expected {:?}",+ weights.data_type(),+ data_type+ )),+ ));+ }++ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };++ Ok(Self {+ kernel,+ weights_buffer,+ vocab_size: vocab_size as u32,+ model_dim: model_dim as u32,+ input_scale: input_scale.unwrap_or(1.0),+ })+ }+}++impl EncodableBlock for FullPrecisionEmbeddingLookup {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);+ let batch_size = state.active_suffix_length();+ let mut token_ids_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = FullPrecisionEmbeddingLookupArguments {+ token_ids_buffer,+ weights_buffer: &self.weights_buffer,+ output_buffer,+ batch_size: batch_size as u32,+ vocab_size: self.vocab_size,+ model_dim: self.model_dim,+ input_scale: self.input_scale,+ };++ self.kernel+ .encode(encoder, args)+ .expect("Failed to encode full precision embedding lookup kernel");++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}+diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rsindex 619eca99e..3c1386abd 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs@@ -1,11 +1,13 @@//! Embedding encodables (quantized and full-precision).mod error;+mod full_precision_lookup;mod full_precision_readout;mod quantized_lookup;mod quantized_readout;pub use error::QuantizedEmbeddingError;+pub use full_precision_lookup::FullPrecisionEmbeddingLookup;pub use full_precision_readout::FullPrecisionEmbeddingReadout;pub use quantized_lookup::QuantizedEmbeddingLookup;pub use quantized_readout::QuantizedEmbeddingReadout;diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metalnew file mode 100644index 000000000..96ffb2c13--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal@@ -0,0 +1,65 @@+#include <metal_stdlib>+using namespace metal;++template <typename T>+[[kernel]] void full_precision_embedding_lookup(+ const device uint64_t* token_ids [[buffer(0)]], // [batch_size]+ const device T* weights [[buffer(1)]], // [vocab_size, model_dim]+ device T* output [[buffer(2)]], // [batch_size, model_dim]+ constant uint32_t& batch_size [[buffer(3)]],+ constant uint32_t& vocab_size [[buffer(4)]],+ constant uint32_t& model_dim [[buffer(5)]],+ constant float& input_scale [[buffer(6)]],+ uint thread_position_in_grid [[thread_position_in_grid]]+) {+ const uint batch_idx = thread_position_in_grid / model_dim;+ const uint dim_idx = thread_position_in_grid % model_dim;++ if (batch_idx >= batch_size) return;++ const uint64_t token_id = token_ids[batch_idx];+ if (token_id >= vocab_size) {+ output[thread_position_in_grid] = T(0);+ return;+ }++ T value = weights[token_id * model_dim + dim_idx];+ output[thread_position_in_grid] = value * T(input_scale);+}++template [[host_name("full_precision_embedding_lookup_f32")]] [[kernel]] void+full_precision_embedding_lookup<float>(+ const device uint64_t* token_ids [[buffer(0)]],+ const device float* weights [[buffer(1)]],+ device float* output [[buffer(2)]],+ constant uint32_t& batch_size [[buffer(3)]],+ constant uint32_t& vocab_size [[buffer(4)]],+ constant uint32_t& model_dim [[buffer(5)]],+ constant float& input_scale [[buffer(6)]],+ uint thread_position_in_grid [[thread_position_in_grid]]+);++template [[host_name("full_precision_embedding_lookup_f16")]] [[kernel]] void+full_precision_embedding_lookup<half>(+ const device uint64_t* token_ids [[buffer(0)]],+ const device half* weights [[buffer(1)]],+ device half* output [[buffer(2)]],+ constant uint32_t& batch_size [[buffer(3)]],+ constant uint32_t& vocab_size [[buffer(4)]],+ constant uint32_t& model_dim [[buffer(5)]],+ constant float& input_scale [[buffer(6)]],+ uint thread_position_in_grid [[thread_position_in_grid]]+);++template [[host_name("full_precision_embedding_lookup_bf16")]] [[kernel]] void+full_precision_embedding_lookup<bfloat>(+ const device uint64_t* token_ids [[buffer(0)]],+ const device bfloat* weights [[buffer(1)]],+ device bfloat* output [[buffer(2)]],+ constant uint32_t& batch_size [[buffer(3)]],+ constant uint32_t& vocab_size [[buffer(4)]],+ constant uint32_t& model_dim [[buffer(5)]],+ constant float& input_scale [[buffer(6)]],+ uint thread_position_in_grid [[thread_position_in_grid]]+);+diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rsindex 4adbc459a..647eff7f4 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs@@ -11,6 +11,98 @@ use crate::{config::QuantizationMode,};+// ---- Full Precision Embedding Lookup Kernel ----++pub struct FullPrecisionEmbeddingLookupKernel {+ pipeline: MTLComputePipelineState,+}++#[derive(Debug)]+pub struct FullPrecisionEmbeddingLookupArguments<'a> {+ pub token_ids_buffer: &'a MTLBuffer, // [batch_size] as U64+ pub weights_buffer: &'a MTLBuffer, // [vocab_size, model_dim]+ pub output_buffer: &'a MTLBuffer, // [batch_size, model_dim]+ pub batch_size: u32,+ pub vocab_size: u32,+ pub model_dim: u32,+ pub input_scale: f32,+}++impl FullPrecisionEmbeddingLookupKernel {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ ) -> Result<Self, QuantizedEmbeddingError> {+ let dtype_suffix = match data_type {+ DataType::F32 => "f32",+ DataType::F16 => "f16",+ DataType::BF16 => "bf16",+ other => {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ other,+ ));+ },+ };+ let kernel_name =+ format!("full_precision_embedding_lookup_{}", dtype_suffix);++ let (pipeline, _) = mtl_context+ .compute_pipeline_state_with_reflection(&kernel_name, None)+ .map_err(QuantizedEmbeddingError::MetalError)?;++ Ok(Self {+ pipeline,+ })+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ args: FullPrecisionEmbeddingLookupArguments,+ ) -> Result<(), QuantizedEmbeddingError> {+ encoder.set_compute_pipeline_state(&self.pipeline);++ encoder.set_buffer(0, Some(args.token_ids_buffer), 0);+ encoder.set_buffer(1, Some(args.weights_buffer), 0);+ encoder.set_buffer(2, Some(args.output_buffer), 0);++ encoder.set_bytes(+ 3,+ size_of::<u32>() as u64,+ &args.batch_size as *const u32 as *const _,+ );+ encoder.set_bytes(+ 4,+ size_of::<u32>() as u64,+ &args.vocab_size as *const u32 as *const _,+ );+ encoder.set_bytes(+ 5,+ size_of::<u32>() as u64,+ &args.model_dim as *const u32 as *const _,+ );+ encoder.set_bytes(+ 6,+ size_of::<f32>() as u64,+ &args.input_scale as *const f32 as *const _,+ );++ let total_threads = (args.batch_size * args.model_dim) as u64;+ let threads_per_threadgroup = 256u64;+ let threadgroups = (total_threads + threads_per_threadgroup - 1)+ / threads_per_threadgroup;++ encoder.dispatch_thread_groups(+ MTLSize::new(threadgroups, 1, 1),+ MTLSize::new(threads_per_threadgroup, 1, 1),+ );++ Ok(())+ }+}++// ---- Quantized Embedding Lookup Kernel ----+pub struct QuantizedEmbeddingLookupKernel {pipeline: MTLComputePipelineState,}