A high-performance inference engine for AI models
use input scale
eugenee0236afb3ecc
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding.rs b/crates/uzu/src/backends/metal/encodable_block/embedding.rsindex 23ac3698a..1b22fde31 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding.rs@@ -42,6 +42,7 @@ pub struct QuantizedEmbeddingLookup {vocab_size: u32,model_dim: u32,group_size: u32,+ input_scale: f32,}impl QuantizedEmbeddingLookup {@@ -52,6 +53,7 @@ impl QuantizedEmbeddingLookup {model_dim: usize,group_size: usize,mode: QuantizationMode,+ input_scale: f32,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Result<Self, QuantizedEmbeddingError> {let packing_divisor = mode.packing_divisor();@@ -179,6 +181,7 @@ impl QuantizedEmbeddingLookup {vocab_size: vocab_size as u32,model_dim: model_dim as u32,group_size: group_size as u32,+ input_scale,})}}@@ -211,6 +214,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {vocab_size: self.vocab_size,model_dim: self.model_dim,group_size: self.group_size,+ input_scale: self.input_scale,};self.kerneldiff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex b530c06bc..79edd8901 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -472,6 +472,8 @@ pub fn embed_block(} => {let data_type: DataType =config.output_norm_config.scale_precision.into();+ let input_scale =+ config.embedding_config.common().input_scale.unwrap_or(1.0);let embeddings_tree = parameter_tree.subtree("embedding")@@ -484,6 +486,7 @@ pub fn embed_block(config.model_dim,*group_size,*embedding_quantization_mode,+ input_scale,&embeddings_tree,).expect("Failed to create quantized embedding lookup kernel");diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rsindex 499a22776..4adbc459a 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs@@ -26,6 +26,7 @@ pub struct QuantizedEmbeddingLookupArguments<'a> {pub vocab_size: u32,pub model_dim: u32,pub group_size: u32,+ pub input_scale: f32,}impl QuantizedEmbeddingLookupKernel {@@ -98,6 +99,11 @@ impl QuantizedEmbeddingLookupKernel {size_of::<u32>() as u64,&args.group_size as *const u32 as *const _,);+ encoder.set_bytes(+ 9,+ size_of::<f32>() as u64,+ &args.input_scale as *const f32 as *const _,+ );// Dispatch one thread per output elementlet total_threads = (args.batch_size * args.model_dim) as u64;diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex a1e2f9e8a..071e540e9 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -13,6 +13,7 @@ template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]) {const uint batch_idx = thread_position_in_grid / model_dim;@@ -53,7 +54,9 @@ template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>SIGNED_STORAGE ? int(weights_i8[elem_idx]) : int(weights_u8[elem_idx]);}- output[thread_position_in_grid] = scale * T(quantized_value) + bias;+ float out_f = float(scale) * float(quantized_value) + float(bias);+ out_f *= input_scale;+ output[thread_position_in_grid] = T(out_f);}template [[host_name("quantized_embedding_lookup_f32_uint4")]] [[kernel]] void@@ -67,6 +70,7 @@ quantized_embedding_lookup<float, 2, false>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -81,6 +85,7 @@ quantized_embedding_lookup<half, 2, false>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -95,6 +100,7 @@ quantized_embedding_lookup<bfloat, 2, false>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -109,6 +115,7 @@ quantized_embedding_lookup<float, 1, true>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -123,6 +130,7 @@ quantized_embedding_lookup<half, 1, true>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -137,6 +145,7 @@ quantized_embedding_lookup<bfloat, 1, true>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -151,6 +160,7 @@ quantized_embedding_lookup<float, 1, false>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -165,6 +175,7 @@ quantized_embedding_lookup<half, 1, false>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);@@ -179,5 +190,6 @@ quantized_embedding_lookup<bfloat, 1, false>(constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],+ constant float& input_scale [[buffer(9)]],uint thread_position_in_grid [[thread_position_in_grid]]);