Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

use input scale

eugenee0236afb3ecc
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding.rs b/crates/uzu/src/backends/metal/encodable_block/embedding.rs
index 23ac3698a..1b22fde31 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding.rs
@@ -42,6 +42,7 @@ pub struct QuantizedEmbeddingLookup {
vocab_size: u32,
model_dim: u32,
group_size: u32,
+ input_scale: f32,
}
impl QuantizedEmbeddingLookup {
@@ -52,6 +53,7 @@ impl QuantizedEmbeddingLookup {
model_dim: usize,
group_size: usize,
mode: QuantizationMode,
+ input_scale: f32,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Result<Self, QuantizedEmbeddingError> {
let packing_divisor = mode.packing_divisor();
@@ -179,6 +181,7 @@ impl QuantizedEmbeddingLookup {
vocab_size: vocab_size as u32,
model_dim: model_dim as u32,
group_size: group_size as u32,
+ input_scale,
})
}
}
@@ -211,6 +214,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {
vocab_size: self.vocab_size,
model_dim: self.model_dim,
group_size: self.group_size,
+ input_scale: self.input_scale,
};
self.kernel
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index b530c06bc..79edd8901 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -472,6 +472,8 @@ pub fn embed_block(
} => {
let data_type: DataType =
config.output_norm_config.scale_precision.into();
+ let input_scale =
+ config.embedding_config.common().input_scale.unwrap_or(1.0);
let embeddings_tree = parameter_tree
.subtree("embedding")
@@ -484,6 +486,7 @@ pub fn embed_block(
config.model_dim,
*group_size,
*embedding_quantization_mode,
+ input_scale,
&embeddings_tree,
)
.expect("Failed to create quantized embedding lookup kernel");
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
index 499a22776..4adbc459a 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
@@ -26,6 +26,7 @@ pub struct QuantizedEmbeddingLookupArguments<'a> {
pub vocab_size: u32,
pub model_dim: u32,
pub group_size: u32,
+ pub input_scale: f32,
}
impl QuantizedEmbeddingLookupKernel {
@@ -98,6 +99,11 @@ impl QuantizedEmbeddingLookupKernel {
size_of::<u32>() as u64,
&args.group_size as *const u32 as *const _,
);
+ encoder.set_bytes(
+ 9,
+ size_of::<f32>() as u64,
+ &args.input_scale as *const f32 as *const _,
+ );
// Dispatch one thread per output element
let total_threads = (args.batch_size * args.model_dim) as u64;
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index a1e2f9e8a..071e540e9 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -13,6 +13,7 @@ template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
) {
const uint batch_idx = thread_position_in_grid / model_dim;
@@ -53,7 +54,9 @@ template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>
SIGNED_STORAGE ? int(weights_i8[elem_idx]) : int(weights_u8[elem_idx]);
}
- output[thread_position_in_grid] = scale * T(quantized_value) + bias;
+ float out_f = float(scale) * float(quantized_value) + float(bias);
+ out_f *= input_scale;
+ output[thread_position_in_grid] = T(out_f);
}
template [[host_name("quantized_embedding_lookup_f32_uint4")]] [[kernel]] void
@@ -67,6 +70,7 @@ quantized_embedding_lookup<float, 2, false>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -81,6 +85,7 @@ quantized_embedding_lookup<half, 2, false>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -95,6 +100,7 @@ quantized_embedding_lookup<bfloat, 2, false>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -109,6 +115,7 @@ quantized_embedding_lookup<float, 1, true>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -123,6 +130,7 @@ quantized_embedding_lookup<half, 1, true>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -137,6 +145,7 @@ quantized_embedding_lookup<bfloat, 1, true>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -151,6 +160,7 @@ quantized_embedding_lookup<float, 1, false>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -165,6 +175,7 @@ quantized_embedding_lookup<half, 1, false>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);
@@ -179,5 +190,6 @@ quantized_embedding_lookup<bfloat, 1, false>(
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
+ constant float& input_scale [[buffer(9)]],
uint thread_position_in_grid [[thread_position_in_grid]]
);