Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

fix after merge

eugene75a81beabf6b
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
index 9f4099dc3..753c4bb00 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
@@ -24,6 +24,7 @@ pub struct QuantizedEmbeddingLookup {
weights_buffer: MTLBuffer,
scales_buffer: MTLBuffer,
biases_buffer: MTLBuffer,
+ input_scale: f32,
vocab_size: u32,
model_dim: u32,
group_size: u32,
@@ -37,6 +38,7 @@ impl QuantizedEmbeddingLookup {
model_dim: usize,
group_size: usize,
mode: QuantizationMode,
+ input_scale: f32,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Result<Self, EmbeddingError> {
Self::new_with_names(
@@ -46,6 +48,7 @@ impl QuantizedEmbeddingLookup {
model_dim,
group_size,
mode,
+ input_scale,
"weights",
"scales",
"biases",
@@ -60,6 +63,7 @@ impl QuantizedEmbeddingLookup {
model_dim: usize,
group_size: usize,
mode: QuantizationMode,
+ input_scale: f32,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Result<Self, EmbeddingError> {
Self::new_with_names(
@@ -69,6 +73,7 @@ impl QuantizedEmbeddingLookup {
model_dim,
group_size,
mode,
+ input_scale,
"input_weights",
"input_scales",
"input_biases",
@@ -83,6 +88,7 @@ impl QuantizedEmbeddingLookup {
model_dim: usize,
group_size: usize,
mode: QuantizationMode,
+ input_scale: f32,
weights_name: &str,
scales_name: &str,
biases_name: &str,
@@ -102,13 +108,13 @@ impl QuantizedEmbeddingLookup {
})?;
if weights.data_type() != mode.storage_type() {
- return Err(EmbeddingError::MetalError(
- MTLError::Generic(format!(
+ return Err(EmbeddingError::MetalError(MTLError::Generic(
+ format!(
"Expected packed weights of type {:?}, got {:?}",
mode.storage_type(),
weights.data_type()
- )),
- ));
+ ),
+ )));
}
// Load scales [vocab_size, num_groups]
@@ -122,24 +128,24 @@ impl QuantizedEmbeddingLookup {
// Validate shapes and types
let num_groups = (model_dim + group_size - 1) / group_size;
if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(EmbeddingError::MetalError(
- MTLError::Generic(format!(
+ return Err(EmbeddingError::MetalError(MTLError::Generic(
+ format!(
"Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
weights.shape(),
vocab_size,
model_dim / packing_divisor
- )),
- ));
+ ),
+ )));
}
if scales.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(
- MTLError::Generic(format!(
+ return Err(EmbeddingError::MetalError(MTLError::Generic(
+ format!(
"Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
scales.shape(),
vocab_size,
num_groups
- )),
- ));
+ ),
+ )));
}
if scales.data_type() != data_type {
return Err(EmbeddingError::UnsupportedDataType(
@@ -151,14 +157,14 @@ impl QuantizedEmbeddingLookup {
let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {
Ok(mut deq_biases) => {
if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(
- MTLError::Generic(format!(
+ return Err(EmbeddingError::MetalError(MTLError::Generic(
+ format!(
"Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",
deq_biases.shape(),
vocab_size,
num_groups
- )),
- ));
+ ),
+ )));
}
if deq_biases.data_type() != data_type {
return Err(EmbeddingError::UnsupportedDataType(
@@ -172,9 +178,7 @@ impl QuantizedEmbeddingLookup {
DataType::F16 | DataType::BF16 => 2,
DataType::F32 => 4,
other => {
- return Err(
- EmbeddingError::UnsupportedDataType(other),
- );
+ return Err(EmbeddingError::UnsupportedDataType(other));
},
};
let size_bytes = (vocab_size * num_groups * elem_size) as u64;
@@ -201,6 +205,7 @@ impl QuantizedEmbeddingLookup {
weights_buffer,
scales_buffer,
biases_buffer,
+ input_scale,
vocab_size: vocab_size as u32,
model_dim: model_dim as u32,
group_size: group_size as u32,
@@ -235,6 +240,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {
vocab_size: self.vocab_size,
model_dim: self.model_dim,
group_size: self.group_size,
+ input_scale: self.input_scale,
};
self.kernel
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
index e3cbf1a28..76baf7024 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -2,7 +2,7 @@
use metal::CommandBufferRef;
-use super::{EncodableBlock, EncodingParameters, QuantizedLinear};
+use super::{EncodableBlock, EncodingParameters};
use crate::backends::metal::{
forward_pass::{ArrayId, ForwardPassState},
kernel::mlp::MlpGateActMulEncodable,
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index f3c280e2e..f8a6f3229 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -148,21 +148,15 @@ impl ShortConvMixer {
command_buffer: &CommandBufferRef,
suffix_length: usize,
) {
- let padded = state
- .short_conv_padded_buffer()
- .expect("short_conv_padded scratch buffer not initialized");
-
let arrays = state.arrays(&[
ArrayId::SsmInProj,
ArrayId::ShortConvState(self.layer_index),
ArrayId::AttentionOutput,
]);
- let mut padded = padded.borrow_mut();
let mut in_proj = arrays[0].borrow_mut();
let mut conv_state = arrays[1].borrow_mut();
let mut out = arrays[2].borrow_mut();
- let padded_buf = unsafe { padded.mtl_buffer().to_owned() };
let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };
let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };
let out_buf = unsafe { out.mtl_buffer().to_owned() };
@@ -189,9 +183,7 @@ impl ShortConvMixer {
metal::MTLResourceOptions::StorageModePrivate,
);
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- let compute = mtl_command_buffer.new_compute_command_encoder();
+ let compute = command_buffer.new_compute_command_encoder();
self.short_conv_kernel
.encode_pack(
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index d26cf38b1..ea855f24a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -242,6 +242,8 @@ pub fn embed_block(
..
} => {
let data_type: DataType = (*activation_precision).into();
+ let input_scale =
+ config.embedding_config.common().input_scale.unwrap_or(1.0);
let embeddings_tree = parameter_tree
.subtree("embedding")
@@ -254,6 +256,7 @@ pub fn embed_block(
config.model_dim,
*group_size,
*embedding_quantization_mode,
+ input_scale,
&embeddings_tree,
)
.expect("Failed to create quantized embedding lookup kernel");
@@ -273,40 +276,15 @@ pub fn embed_block(
// For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dim
let group_size = config.model_dim;
-
- let block = QuantizedEmbeddingLookup::new_tied(
- context,
- data_type,
- config.vocab_size,
- config.model_dim,
- group_size,
- *embedding_quantization_mode,
- &embeddings_tree,
- )
- .expect("Failed to create quantized embedding lookup kernel");
-
- Box::new(block)
- },
- EmbeddingConfig::MLXQuantizedTied {
- group_size,
- embedding_quantization_mode,
- ..
- } => {
- let data_type: DataType =
- config.output_norm_config.scale_precision.into();
let input_scale =
config.embedding_config.common().input_scale.unwrap_or(1.0);
- let embeddings_tree = parameter_tree
- .subtree("embedding")
- .expect("Failed to get embedding subtree");
-
let block = QuantizedEmbeddingLookup::new_tied(
context,
data_type,
config.vocab_size,
config.model_dim,
- *group_size,
+ group_size,
*embedding_quantization_mode,
input_scale,
&embeddings_tree,
@@ -315,7 +293,7 @@ pub fn embed_block(
Box::new(block)
},
- EmbeddingConfig::MLXQuantizedUntied {
+ EmbeddingConfig::MLXQuantizedTied {
group_size,
embedding_quantization_mode,
..
@@ -329,7 +307,7 @@ pub fn embed_block(
.subtree("embedding")
.expect("Failed to get embedding subtree");
- let block = QuantizedEmbeddingLookup::new_untied_input(
+ let block = QuantizedEmbeddingLookup::new_tied(
context,
data_type,
config.vocab_size,
@@ -483,8 +461,8 @@ pub fn readout_block(
data_type,
config.vocab_size,
config.model_dim,
- group_size,
- embedding_quantization_mode,
+ *group_size,
+ *embedding_quantization_mode,
&embeddings_tree,
)
.expect("Failed to create quantized embedding readout kernel");
diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rs
index 9ed78fd5b..490d76e37 100644
--- a/crates/uzu/src/config/embedding.rs
+++ b/crates/uzu/src/config/embedding.rs
@@ -58,15 +58,6 @@ pub enum EmbeddingConfig {
activation_quantization_mode: Option<QuantizationMode>,
activation_precision: ConfigDataType,
},
- #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]
- MLXQuantizedUntied {
- #[serde(flatten)]
- common: EmbeddingConfigCommon,
- group_size: usize,
- embedding_quantization_mode: QuantizationMode,
- activation_quantization_mode: Option<QuantizationMode>,
- activation_precision: ConfigDataType,
- },
}
impl EmbeddingConfig {
@@ -96,10 +87,6 @@ impl EmbeddingConfig {
common,
..
} => common,
- EmbeddingConfig::MLXQuantizedUntied {
- common,
- ..
- } => common,
}
}
}