Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

full precision lookup

eugeneec6f03e1b4f2
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
new file mode 100644
index 000000000..5bd6c6ae1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_lookup.rs
@@ -0,0 +1,128 @@
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer;
+
+use super::QuantizedEmbeddingError;
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::embedding::{
+ FullPrecisionEmbeddingLookupArguments,
+ FullPrecisionEmbeddingLookupKernel,
+ },
+ },
+ parameters::ParameterTree,
+};
+
+use super::super::{EncodableBlock, EncodingParameters};
+
+pub struct FullPrecisionEmbeddingLookup {
+ kernel: FullPrecisionEmbeddingLookupKernel,
+ weights_buffer: MTLBuffer,
+ vocab_size: u32,
+ model_dim: u32,
+ input_scale: f32,
+}
+
+impl FullPrecisionEmbeddingLookup {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ input_scale: Option<f32>,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ let kernel =
+ FullPrecisionEmbeddingLookupKernel::new(mtl_context, data_type)?;
+
+ let mut weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => parameter_tree.leaf("input_weights").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ if weights.shape() != [vocab_size, model_dim] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding lookup weights shape mismatch: got {:?}, \
+ expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim
+ )),
+ ));
+ }
+
+ if weights.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Weights dtype mismatch: got {:?}, expected {:?}",
+ weights.data_type(),
+ data_type
+ )),
+ ));
+ }
+
+ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
+
+ Ok(Self {
+ kernel,
+ weights_buffer,
+ vocab_size: vocab_size as u32,
+ model_dim: model_dim as u32,
+ input_scale: input_scale.unwrap_or(1.0),
+ })
+ }
+}
+
+impl EncodableBlock for FullPrecisionEmbeddingLookup {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
+ let batch_size = state.active_suffix_length();
+ let mut token_ids_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = FullPrecisionEmbeddingLookupArguments {
+ token_ids_buffer,
+ weights_buffer: &self.weights_buffer,
+ output_buffer,
+ batch_size: batch_size as u32,
+ vocab_size: self.vocab_size,
+ model_dim: self.model_dim,
+ input_scale: self.input_scale,
+ };
+
+ self.kernel
+ .encode(encoder, args)
+ .expect("Failed to encode full precision embedding lookup kernel");
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
+
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
index 619eca99e..3c1386abd 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
@@ -1,11 +1,13 @@
//! Embedding encodables (quantized and full-precision).
mod error;
+mod full_precision_lookup;
mod full_precision_readout;
mod quantized_lookup;
mod quantized_readout;
pub use error::QuantizedEmbeddingError;
+pub use full_precision_lookup::FullPrecisionEmbeddingLookup;
pub use full_precision_readout::FullPrecisionEmbeddingReadout;
pub use quantized_lookup::QuantizedEmbeddingLookup;
pub use quantized_readout::QuantizedEmbeddingReadout;
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
new file mode 100644
index 000000000..96ffb2c13
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
@@ -0,0 +1,65 @@
+#include <metal_stdlib>
+using namespace metal;
+
+template <typename T>
+[[kernel]] void full_precision_embedding_lookup(
+ const device uint64_t* token_ids [[buffer(0)]], // [batch_size]
+ const device T* weights [[buffer(1)]], // [vocab_size, model_dim]
+ device T* output [[buffer(2)]], // [batch_size, model_dim]
+ constant uint32_t& batch_size [[buffer(3)]],
+ constant uint32_t& vocab_size [[buffer(4)]],
+ constant uint32_t& model_dim [[buffer(5)]],
+ constant float& input_scale [[buffer(6)]],
+ uint thread_position_in_grid [[thread_position_in_grid]]
+) {
+ const uint batch_idx = thread_position_in_grid / model_dim;
+ const uint dim_idx = thread_position_in_grid % model_dim;
+
+ if (batch_idx >= batch_size) return;
+
+ const uint64_t token_id = token_ids[batch_idx];
+ if (token_id >= vocab_size) {
+ output[thread_position_in_grid] = T(0);
+ return;
+ }
+
+ T value = weights[token_id * model_dim + dim_idx];
+ output[thread_position_in_grid] = value * T(input_scale);
+}
+
+template [[host_name("full_precision_embedding_lookup_f32")]] [[kernel]] void
+full_precision_embedding_lookup<float>(
+ const device uint64_t* token_ids [[buffer(0)]],
+ const device float* weights [[buffer(1)]],
+ device float* output [[buffer(2)]],
+ constant uint32_t& batch_size [[buffer(3)]],
+ constant uint32_t& vocab_size [[buffer(4)]],
+ constant uint32_t& model_dim [[buffer(5)]],
+ constant float& input_scale [[buffer(6)]],
+ uint thread_position_in_grid [[thread_position_in_grid]]
+);
+
+template [[host_name("full_precision_embedding_lookup_f16")]] [[kernel]] void
+full_precision_embedding_lookup<half>(
+ const device uint64_t* token_ids [[buffer(0)]],
+ const device half* weights [[buffer(1)]],
+ device half* output [[buffer(2)]],
+ constant uint32_t& batch_size [[buffer(3)]],
+ constant uint32_t& vocab_size [[buffer(4)]],
+ constant uint32_t& model_dim [[buffer(5)]],
+ constant float& input_scale [[buffer(6)]],
+ uint thread_position_in_grid [[thread_position_in_grid]]
+);
+
+template [[host_name("full_precision_embedding_lookup_bf16")]] [[kernel]] void
+full_precision_embedding_lookup<bfloat>(
+ const device uint64_t* token_ids [[buffer(0)]],
+ const device bfloat* weights [[buffer(1)]],
+ device bfloat* output [[buffer(2)]],
+ constant uint32_t& batch_size [[buffer(3)]],
+ constant uint32_t& vocab_size [[buffer(4)]],
+ constant uint32_t& model_dim [[buffer(5)]],
+ constant float& input_scale [[buffer(6)]],
+ uint thread_position_in_grid [[thread_position_in_grid]]
+);
+
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
index 4adbc459a..647eff7f4 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
@@ -11,6 +11,98 @@ use crate::{
config::QuantizationMode,
};
+// ---- Full Precision Embedding Lookup Kernel ----
+
+pub struct FullPrecisionEmbeddingLookupKernel {
+ pipeline: MTLComputePipelineState,
+}
+
+#[derive(Debug)]
+pub struct FullPrecisionEmbeddingLookupArguments<'a> {
+ pub token_ids_buffer: &'a MTLBuffer, // [batch_size] as U64
+ pub weights_buffer: &'a MTLBuffer, // [vocab_size, model_dim]
+ pub output_buffer: &'a MTLBuffer, // [batch_size, model_dim]
+ pub batch_size: u32,
+ pub vocab_size: u32,
+ pub model_dim: u32,
+ pub input_scale: f32,
+}
+
+impl FullPrecisionEmbeddingLookupKernel {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ let dtype_suffix = match data_type {
+ DataType::F32 => "f32",
+ DataType::F16 => "f16",
+ DataType::BF16 => "bf16",
+ other => {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ other,
+ ));
+ },
+ };
+ let kernel_name =
+ format!("full_precision_embedding_lookup_{}", dtype_suffix);
+
+ let (pipeline, _) = mtl_context
+ .compute_pipeline_state_with_reflection(&kernel_name, None)
+ .map_err(QuantizedEmbeddingError::MetalError)?;
+
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ args: FullPrecisionEmbeddingLookupArguments,
+ ) -> Result<(), QuantizedEmbeddingError> {
+ encoder.set_compute_pipeline_state(&self.pipeline);
+
+ encoder.set_buffer(0, Some(args.token_ids_buffer), 0);
+ encoder.set_buffer(1, Some(args.weights_buffer), 0);
+ encoder.set_buffer(2, Some(args.output_buffer), 0);
+
+ encoder.set_bytes(
+ 3,
+ size_of::<u32>() as u64,
+ &args.batch_size as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 4,
+ size_of::<u32>() as u64,
+ &args.vocab_size as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 5,
+ size_of::<u32>() as u64,
+ &args.model_dim as *const u32 as *const _,
+ );
+ encoder.set_bytes(
+ 6,
+ size_of::<f32>() as u64,
+ &args.input_scale as *const f32 as *const _,
+ );
+
+ let total_threads = (args.batch_size * args.model_dim) as u64;
+ let threads_per_threadgroup = 256u64;
+ let threadgroups = (total_threads + threads_per_threadgroup - 1)
+ / threads_per_threadgroup;
+
+ encoder.dispatch_thread_groups(
+ MTLSize::new(threadgroups, 1, 1),
+ MTLSize::new(threads_per_threadgroup, 1, 1),
+ );
+
+ Ok(())
+ }
+}
+
+// ---- Quantized Embedding Lookup Kernel ----
+
pub struct QuantizedEmbeddingLookupKernel {
pipeline: MTLComputePipelineState,
}